dazed/DazedTL
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Format HELL

Browse source
This commit is contained in:
DazedAnon 2024-09-27 11:06:26 -05:00
parent 31c8007055
commit 9fac989f7a
21 changed files with 6392 additions and 4273 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

335
modules/alice.py
View file

@ -15,52 +15,53 @@ from tqdm import tqdm
# Open AI # Open AI
load_dotenv() load_dotenv()
if os.getenv('api').replace(' ', '') != '': if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv('api') openai.base_url = os.getenv("api")
openai.organization = os.getenv('org') openai.organization = os.getenv("org")
openai.api_key = os.getenv('key') openai.api_key = os.getenv("key")
#Globals # Globals
MODEL = os.getenv('model') MODEL = os.getenv("model")
TIMEOUT = int(os.getenv('timeout')) TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv('language').capitalize() LANGUAGE = os.getenv("language").capitalize()
PROMPT = Path('prompt.txt').read_text(encoding='utf-8') PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path('vocab.txt').read_text(encoding='utf-8') VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv('threads')) THREADS = int(os.getenv("threads"))
LOCK = threading.Lock() LOCK = threading.Lock()
WIDTH = int(os.getenv('width')) WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv('listWidth')) LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70 NOTEWIDTH = 70
MAXHISTORY = 10 MAXHISTORY = 10
ESTIMATE = '' ESTIMATE = ""
TOKENS = [0, 0] TOKENS = [0, 0]
NAMESLIST = [] NAMESLIST = []
NAMES = False # Output a list of all the character names found NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = True # Overwrites textwrap FIXTEXTWRAP = True # Overwrites textwrap
IGNORETLTEXT = False # Ignores all translated text. IGNORETLTEXT = False # Ignores all translated text.
MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
#tqdm Globals # tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0 POSITION = 0
LEAVE = False LEAVE = False
# Pricing - Depends on the model https://openai.com/pricing # Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request # Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size. # If you are getting a MISMATCH LENGTH error, lower the batch size.
if 'gpt-3.5' in MODEL: if "gpt-3.5" in MODEL:
INPUTAPICOST = .002 INPUTAPICOST = 0.002
OUTPUTAPICOST = .002 OUTPUTAPICOST = 0.002
BATCHSIZE = 10 BATCHSIZE = 10
elif 'gpt-4' in MODEL: elif "gpt-4" in MODEL:
INPUTAPICOST = .01 INPUTAPICOST = 0.01
OUTPUTAPICOST = .03 OUTPUTAPICOST = 0.03
BATCHSIZE = 1 BATCHSIZE = 1
def handleAlice(filename, estimate): def handleAlice(filename, estimate):
global ESTIMATE global ESTIMATE
totalTokens = [0,0] totalTokens = [0, 0]
ESTIMATE = estimate ESTIMATE = estimate
if estimate: if estimate:
@ -75,17 +76,19 @@ def handleAlice(filename, estimate):
totalTokens[1] += translatedData[1][1] totalTokens[1] += translatedData[1][1]
# Print Total # Print Total
totalString = getResultString(['', totalTokens, None], end - start, 'TOTAL') totalString = getResultString(["", totalTokens, None], end - start, "TOTAL")
# Print any errors on maps # Print any errors on maps
if len(MISMATCH) > 0: if len(MISMATCH) > 0:
return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET return (
totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
)
else: else:
return totalString return totalString
else: else:
try: try:
with open('translated/' + filename, 'w', encoding='UTF-8') as outFile: with open("translated/" + filename, "w", encoding="UTF-8") as outFile:
start = time.time() start = time.time()
translatedData = openFiles(filename) translatedData = openFiles(filename)
@ -98,29 +101,43 @@ def handleAlice(filename, estimate):
totalTokens[1] += translatedData[1][1] totalTokens[1] += translatedData[1][1]
except Exception: except Exception:
traceback.print_exc() traceback.print_exc()
return 'Fail' return "Fail"
return getResultString(["", totalTokens, None], end - start, "TOTAL")
return getResultString(['', totalTokens, None], end - start, 'TOTAL')
def openFiles(filename): def openFiles(filename):
with open('files/' + filename, 'r', encoding='UTF-8') as f: with open("files/" + filename, "r", encoding="UTF-8") as f:
translatedData = parseText(f, filename) translatedData = parseText(f, filename)
return translatedData return translatedData
def getResultString(translatedData, translationTime, filename): def getResultString(translatedData, translationTime, filename):
# File Print String # File Print String
totalTokenstring =\ totalTokenstring = (
Fore.YELLOW +\ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
'[Input: ' + str(translatedData[1][0]) + ']'\ "[Output: "
'[Output: ' + str(translatedData[1][1]) + ']'\ + str(translatedData[1][1])
'[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\ + "]" "[Cost: ${:,.4f}".format(
(translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']' (translatedData[1][0] * 0.001 * INPUTAPICOST)
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
)
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None: if translatedData[2] == None:
# Success # Success
return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET return (
filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.GREEN
+ " \u2713 "
+ Fore.RESET
)
else: else:
# Fail # Fail
@ -129,19 +146,30 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
errorString = str(e) + Fore.RED errorString = str(e) + Fore.RED
return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\ return (
errorString + Fore.RESET filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.RED
+ " \u2717 "
+ errorString
+ Fore.RESET
)
def parseText(data, filename): def parseText(data, filename):
# Get total for progress bar # Get total for progress bar
linesList = data.readlines() linesList = data.readlines()
totalTokens = [0, 0] totalTokens = [0, 0]
totalLines = len(linesList) totalLines = len(linesList)
global LOCK global LOCK
with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: with tqdm(
pbar.desc=filename bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE
pbar.total=totalLines ) as pbar:
pbar.desc = filename
pbar.total = totalLines
try: try:
result = translateLines(linesList, pbar) result = translateLines(linesList, pbar)
totalTokens[0] += result[1][0] totalTokens[0] += result[1][0]
@ -151,6 +179,7 @@ def parseText(data, filename):
return [linesList, totalTokens, e] return [linesList, totalTokens, e]
return [linesList, totalTokens, None] return [linesList, totalTokens, None]
# Grab scenario data from text file # Grab scenario data from text file
def translateLines(linesList, pbar): def translateLines(linesList, pbar):
currentGroup = [] currentGroup = []
@ -165,54 +194,67 @@ def translateLines(linesList, pbar):
try: try:
while i < len(linesList): while i < len(linesList):
# Check if Proper Message # Check if Proper Message
match = re.findall(r's\[[0-9]+\] = \"(.*)\"', linesList[i]) match = re.findall(r"s\[[0-9]+\] = \"(.*)\"", linesList[i])
if len(match) > 0: if len(match) > 0:
jaString = match[0] jaString = match[0]
# Skip Files # Skip Files
if '／' in jaString: if "／" in jaString:
i += 1 i += 1
continue continue
### Translate ### Translate
# Remove any textwrap # Remove any textwrap
jaString = re.sub(r'\\n', ' ', jaString) jaString = re.sub(r"\\n", " ", jaString)
# Grab Speaker # Grab Speaker
speakerMatch = re.findall(r's\[[0-9]+\] = \"([^／]+)\"', linesList[i-1]) speakerMatch = re.findall(
r"s\[[0-9]+\] = \"([^／]+)\"", linesList[i - 1]
)
if len(speakerMatch) > 0: if len(speakerMatch) > 0:
# If there isn't any Japanese in the text just skip # If there isn't any Japanese in the text just skip
if re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', jaString) and '_' not in speakerMatch[0]: if (
re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", jaString)
and "_" not in speakerMatch[0]
):
speaker = speakerMatch[0] speaker = speakerMatch[0]
else: else:
speaker = '' speaker = ""
else: else:
speaker = '' speaker = ""
# Grab rest of the messages # Grab rest of the messages
currentGroup.append(jaString) currentGroup.append(jaString)
# Check if next line should be merged # Check if next line should be merged
if insertBool is True: if insertBool is True:
linesList[i] = re.sub(r'(s\[[0-9]+\]) = \"(.+)\"', r'\1 = ""', linesList[i]) linesList[i] = re.sub(
linesList[i] = linesList[i].replace(';', '') r"(s\[[0-9]+\]) = \"(.+)\"", r'\1 = ""', linesList[i]
)
linesList[i] = linesList[i].replace(";", "")
start = i start = i
while (len(linesList) > i+1 and re.search(r's\[[0-9]+\] = \"\s+(.*)\"', linesList[i+1]) != None): while (
len(linesList) > i + 1
and re.search(r"s\[[0-9]+\] = \"\s+(.*)\"", linesList[i + 1])
!= None
):
multiLine = True multiLine = True
i += 1 i += 1
match = re.findall(r's\[[0-9]+\] = \"\s+(.*)\"', linesList[i]) match = re.findall(r"s\[[0-9]+\] = \"\s+(.*)\"", linesList[i])
currentGroup.append(match[0]) currentGroup.append(match[0])
if insertBool is True: if insertBool is True:
linesList[i] = re.sub(r'(s\[[0-9]+\]) = \"\s+(.+)\"', r'\1 = ""', linesList[i]) linesList[i] = re.sub(
linesList[i] = linesList[i].replace(';', '') r"(s\[[0-9]+\]) = \"\s+(.+)\"", r'\1 = ""', linesList[i]
)
linesList[i] = linesList[i].replace(";", "")
i += 1 i += 1
# Combine Groups and Add Speaker # Combine Groups and Add Speaker
finalJAString = ' '.join(currentGroup) finalJAString = " ".join(currentGroup)
if speaker != '': if speaker != "":
finalJAString = f'{speaker}: {finalJAString}' finalJAString = f"{speaker}: {finalJAString}"
else: else:
finalJAString = f'{finalJAString}' finalJAString = f"{finalJAString}"
# [Passthrough 1] Pulling From File # [Passthrough 1] Pulling From File
if insertBool is False: if insertBool is False:
@ -235,7 +277,7 @@ def translateLines(linesList, pbar):
# Mismatch # Mismatch
else: else:
pbar.write(f'Mismatch: {batchStartIndex} - {i}') pbar.write(f"Mismatch: {batchStartIndex} - {i}")
MISMATCH.append(batch) MISMATCH.append(batch)
batchStartIndex = i batchStartIndex = i
batch.clear() batch.clear()
@ -249,33 +291,41 @@ def translateLines(linesList, pbar):
translatedText = translatedBatch[0] translatedText = translatedBatch[0]
# Remove added speaker and quotes # Remove added speaker and quotes
translatedText = re.sub(r'^.+?:\s', '', translatedText) translatedText = re.sub(r"^.+?:\s", "", translatedText)
# Textwrap # Textwrap
translatedText = translatedText.replace('\"', '\\"') translatedText = translatedText.replace('"', '\\"')
translatedText = textwrap.fill(translatedText, width=WIDTH) translatedText = textwrap.fill(translatedText, width=WIDTH)
# Set Data # Set Data
if multiLine: if multiLine:
textList = translatedText.split("\n") textList = translatedText.split("\n")
for t in textList: for t in textList:
translatedText = translatedText.replace(';', '') translatedText = translatedText.replace(";", "")
translatedText = re.sub(r'(s\[[0-9]+\]) = \"(.*)\"', rf'\1 = "{t}"', linesList[start]) translatedText = re.sub(
translatedText = translatedText.replace(';', '') r"(s\[[0-9]+\]) = \"(.*)\"",
rf'\1 = "{t}"',
linesList[start],
)
translatedText = translatedText.replace(";", "")
linesList[start] = translatedText linesList[start] = translatedText
pbar.update(1) pbar.update(1)
start += 1 start += 1
multiLine = False multiLine = False
translatedText = translatedText.replace(';', '') translatedText = translatedText.replace(";", "")
translatedBatch.pop(0) translatedBatch.pop(0)
else: else:
# Remove any textwrap # Remove any textwrap
translatedText = translatedText.replace('\n', ' ') translatedText = translatedText.replace("\n", " ")
translatedText = re.sub(r'(s\[[0-9]+\]) = \"(.*)\"', rf'\1 = "{translatedText}"', linesList[start]) translatedText = re.sub(
translatedText = translatedText.replace(';', '') r"(s\[[0-9]+\]) = \"(.*)\"",
rf'\1 = "{translatedText}"',
linesList[start],
)
translatedText = translatedText.replace(";", "")
linesList[start] = translatedText linesList[start] = translatedText
pbar.update(1) pbar.update(1)
translatedBatch.pop(0) translatedBatch.pop(0)
# If Batch is empty. Move on. # If Batch is empty. Move on.
if len(translatedBatch) == 0: if len(translatedBatch) == 0:
@ -283,7 +333,7 @@ def translateLines(linesList, pbar):
batchStartIndex = i batchStartIndex = i
pbar.update(1) pbar.update(1)
batch.clear() batch.clear()
currentGroup = [] currentGroup = []
else: else:
if insertBool is True: if insertBool is True:
@ -295,70 +345,72 @@ def translateLines(linesList, pbar):
traceback.print_exc() traceback.print_exc()
return [linesList, tokens] return [linesList, tokens]
def subVars(jaString): def subVars(jaString):
jaString = jaString.replace('\u3000', ' ') jaString = jaString.replace("\u3000", " ")
# Nested # Nested
count = 0 count = 0
nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString) nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList) nestedList = set(nestedList)
if len(nestedList) != 0: if len(nestedList) != 0:
for icon in nestedList: for icon in nestedList:
jaString = jaString.replace(icon, '{Nested_' + str(count) + '}') jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString) iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList) iconList = set(iconList)
if len(iconList) != 0: if len(iconList) != 0:
for icon in iconList: for icon in iconList:
jaString = jaString.replace(icon, '{Ascii_' + str(count) + '}') jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString) colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList) colorList = set(colorList)
if len(colorList) != 0: if len(colorList) != 0:
for color in colorList: for color in colorList:
jaString = jaString.replace(color, '{Color_' + str(count) + '}') jaString = jaString.replace(color, "{Color_" + str(count) + "}")
count += 1 count += 1
# Names # Names
count = 0 count = 0
nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString) nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList) nameList = set(nameList)
if len(nameList) != 0: if len(nameList) != 0:
for name in nameList: for name in nameList:
jaString = jaString.replace(name, '{Noun_' + str(count) + '}') jaString = jaString.replace(name, "{Noun_" + str(count) + "}")
count += 1 count += 1
# Variables # Variables
count = 0 count = 0
varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString) varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList) varList = set(varList)
if len(varList) != 0: if len(varList) != 0:
for var in varList: for var in varList:
jaString = jaString.replace(var, '{Var_' + str(count) + '}') jaString = jaString.replace(var, "{Var_" + str(count) + "}")
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
formatList = re.findall(r'[\\]+[\w]+\[.+?\]', jaString) formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
formatList = set(formatList) formatList = set(formatList)
if len(formatList) != 0: if len(formatList) != 0:
for var in formatList: for var in formatList:
jaString = jaString.replace(var, '{FCode_' + str(count) + '}') jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
count += 1 count += 1
# Put all lists in list and return # Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList] allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList] return [jaString, allList]
def resubVars(translatedText, allList): def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense # Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r'\[\s?.+?\s?\]', translatedText) matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0: if len(matchList) > 0:
for match in matchList: for match in matchList:
text = match.strip() text = match.strip()
@ -368,54 +420,58 @@ def resubVars(translatedText, allList):
count = 0 count = 0
if len(allList[0]) != 0: if len(allList[0]) != 0:
for var in allList[0]: for var in allList[0]:
translatedText = translatedText.replace('{Nested_' + str(count) + '}', var) translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
if len(allList[1]) != 0: if len(allList[1]) != 0:
for var in allList[1]: for var in allList[1]:
translatedText = translatedText.replace('{Ascii_' + str(count) + '}', var) translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
if len(allList[2]) != 0: if len(allList[2]) != 0:
for var in allList[2]: for var in allList[2]:
translatedText = translatedText.replace('{Color_' + str(count) + '}', var) translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
count += 1 count += 1
# Names # Names
count = 0 count = 0
if len(allList[3]) != 0: if len(allList[3]) != 0:
for var in allList[3]: for var in allList[3]:
translatedText = translatedText.replace('{Noun_' + str(count) + '}', var) translatedText = translatedText.replace("{Noun_" + str(count) + "}", var)
count += 1 count += 1
# Vars # Vars
count = 0 count = 0
if len(allList[4]) != 0: if len(allList[4]) != 0:
for var in allList[4]: for var in allList[4]:
translatedText = translatedText.replace('{Var_' + str(count) + '}', var) translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
if len(allList[5]) != 0: if len(allList[5]) != 0:
for var in allList[5]: for var in allList[5]:
translatedText = translatedText.replace('{FCode_' + str(count) + '}', var) translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
count += 1 count += 1
return translatedText return translatedText
def batchList(input_list, batch_size): def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0: if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer") raise ValueError("batch_size must be a positive integer")
return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)] return [
input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
]
def createContext(fullPromptFlag, subbedT): def createContext(fullPromptFlag, subbedT):
characters = 'Game Characters:\n\ characters = "Game Characters:\n\
林つかさ (Tsukasa Hayashi) - Female\n\ 林つかさ (Tsukasa Hayashi) - Female\n\
山田美兎 (Miyato Yamada) - Female\n\ 山田美兎 (Miyato Yamada) - Female\n\
鈴木赤音 (Akane Suzuki) - Female\n\ 鈴木赤音 (Akane Suzuki) - Female\n\
@ -428,13 +484,17 @@ def createContext(fullPromptFlag, subbedT):
モリー・ボイド (Molly Boyd) - Female\n\ モリー・ボイド (Molly Boyd) - Female\n\
オルガ・ブヤチッチ (Olga Buyachich) - Female\n\ オルガ・ブヤチッチ (Olga Buyachich) - Female\n\
アッチャラー ギッティ (Atchara Gitti) - Female\n\ アッチャラー ギッティ (Atchara Gitti) - Female\n\
' "
system = PROMPT if fullPromptFlag else \ system = (
f'Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`' PROMPT
user = f'{subbedT}' if fullPromptFlag
else f"Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`"
)
user = f"{subbedT}"
return characters, system, user return characters, system, user
def translateText(characters, system, user, history): def translateText(characters, system, user, history):
# Prompt # Prompt
msg = [{"role": "system", "content": system + characters}] msg = [{"role": "system", "content": system + characters}]
@ -447,9 +507,9 @@ def translateText(characters, system, user, history):
msg.extend([{"role": "assistant", "content": h} for h in history]) msg.extend([{"role": "assistant", "content": h} for h in history])
else: else:
msg.append({"role": "assistant", "content": history}) msg.append({"role": "assistant", "content": history})
# Content to TL # Content to TL
msg.append({"role": "user", "content": f'{user}'}) msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create( response = openai.chat.completions.create(
temperature=0.1, temperature=0.1,
frequency_penalty=0.1, frequency_penalty=0.1,
@ -458,40 +518,47 @@ def translateText(characters, system, user, history):
) )
return response return response
def cleanTranslatedText(translatedText, varResponse): def cleanTranslatedText(translatedText, varResponse):
placeholders = { placeholders = {
f'{LANGUAGE} Translation: ': '', f"{LANGUAGE} Translation: ": "",
'Translation: ': '', "Translation: ": "",
'っ': '', "っ": "",
'〜': '~', "〜": "~",
'ッ': '', "ッ": "",
'。': '.', "。": ".",
'Placeholder Text': '' "Placeholder Text": "",
# Add more replacements as needed # Add more replacements as needed
} }
for target, replacement in placeholders.items(): for target, replacement in placeholders.items():
translatedText = translatedText.replace(target, replacement) translatedText = translatedText.replace(target, replacement)
translatedText = resubVars(translatedText, varResponse[1]) translatedText = resubVars(translatedText, varResponse[1])
if '\n' in translatedText: if "\n" in translatedText:
return [line for line in translatedText.split('\n') if line] return [line for line in translatedText.split("\n") if line]
else: else:
return [line for line in translatedText.split('\\n') if line] return [line for line in translatedText.split("\\n") if line]
def extractTranslation(translatedTextList, is_list): def extractTranslation(translatedTextList, is_list):
pattern = r'<Line(\d+)>[\\]*`?(.*?)[\\]*?`?</?Line\d+>' pattern = r"<Line(\d+)>[\\]*`?(.*?)[\\]*?`?</?Line\d+>"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item. # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list: if is_list:
return [re.findall(pattern, line)[0][1] for line in translatedTextList if re.search(pattern, line)] return [
re.findall(pattern, line)[0][1]
for line in translatedTextList
if re.search(pattern, line)
]
else: else:
matchList = re.findall(pattern, translatedTextList) matchList = re.findall(pattern, translatedTextList)
return matchList[0][1] if matchList else translatedTextList return matchList[0][1] if matchList else translatedTextList
def countTokens(characters, system, user, history): def countTokens(characters, system, user, history):
inputTotalTokens = 0 inputTotalTokens = 0
outputTotalTokens = 0 outputTotalTokens = 0
enc = tiktoken.encoding_for_model('gpt-4') enc = tiktoken.encoding_for_model("gpt-4")
# Input # Input
if isinstance(history, list): if isinstance(history, list):
for line in history: for line in history:
@ -503,15 +570,17 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user)) inputTotalTokens += len(enc.encode(user))
# Output # Output
outputTotalTokens += round(len(enc.encode(user))*3) outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens] return [inputTotalTokens, outputTotalTokens]
def combineList(tlist, text): def combineList(tlist, text):
if isinstance(text, list): if isinstance(text, list):
return [t for sublist in tlist for t in sublist] return [t for sublist in tlist for t in sublist]
return tlist[0] return tlist[0]
@retry(exceptions=Exception, tries=5, delay=5) @retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag): def translateGPT(text, history, fullPromptFlag):
totalTokens = [0, 0] totalTokens = [0, 0]
@ -523,8 +592,10 @@ def translateGPT(text, history, fullPromptFlag):
for index, tItem in enumerate(tList): for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting # Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list): if isinstance(tItem, list):
payload = '\n'.join([f'<Line{i}>`{item}`</Line{i}>' for i, item in enumerate(tItem)]) payload = "\n".join(
payload = payload.replace('``', '`Placeholder Text`') [f"<Line{i}>`{item}`</Line{i}>" for i, item in enumerate(tItem)]
)
payload = payload.replace("``", "`Placeholder Text`")
varResponse = subVars(payload) varResponse = subVars(payload)
subbedT = varResponse[0] subbedT = varResponse[0]
else: else:
@ -532,7 +603,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0] subbedT = varResponse[0]
# Things to Check before starting translation # Things to Check before starting translation
if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT): if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue continue
# Create Message # Create Message
@ -557,11 +628,13 @@ def translateGPT(text, history, fullPromptFlag):
extractedTranslations = extractTranslation(translatedTextList, True) extractedTranslations = extractTranslation(translatedTextList, True)
tList[index] = extractedTranslations tList[index] = extractedTranslations
if len(tItem) != len(translatedTextList): if len(tItem) != len(translatedTextList):
mismatch = True # Just here so breakpoint can be set mismatch = True # Just here so breakpoint can be set
history = extractedTranslations[-10:] # Update history if we have a list history = extractedTranslations[-10:] # Update history if we have a list
else: else:
# Ensure we're passing a single string to extractTranslation # Ensure we're passing a single string to extractTranslation
extractedTranslations = extractTranslation('\n'.join(translatedTextList), False) extractedTranslations = extractTranslation(
"\n".join(translatedTextList), False
)
tList[index] = extractedTranslations tList[index] = extractedTranslations
finalList = combineList(tList, text) finalList = combineList(tList, text)

288
modules/anim.py
View file

@ -16,52 +16,53 @@ from tqdm import tqdm
# Open AI # Open AI
load_dotenv() load_dotenv()
if os.getenv('api').replace(' ', '') != '': if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv('api') openai.base_url = os.getenv("api")
openai.organization = os.getenv('org') openai.organization = os.getenv("org")
openai.api_key = os.getenv('key') openai.api_key = os.getenv("key")
#Globals # Globals
MODEL = os.getenv('model') MODEL = os.getenv("model")
TIMEOUT = int(os.getenv('timeout')) TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv('language').capitalize() LANGUAGE = os.getenv("language").capitalize()
PROMPT = Path('prompt.txt').read_text(encoding='utf-8') PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path('vocab.txt').read_text(encoding='utf-8') VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv('threads')) THREADS = int(os.getenv("threads"))
LOCK = threading.Lock() LOCK = threading.Lock()
WIDTH = int(os.getenv('width')) WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv('listWidth')) LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70 NOTEWIDTH = 70
MAXHISTORY = 10 MAXHISTORY = 10
ESTIMATE = '' ESTIMATE = ""
TOKENS = [0, 0] TOKENS = [0, 0]
NAMESLIST = [] NAMESLIST = []
NAMES = False # Output a list of all the character names found NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = True # Overwrites textwrap FIXTEXTWRAP = True # Overwrites textwrap
IGNORETLTEXT = False # Ignores all translated text. IGNORETLTEXT = False # Ignores all translated text.
MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
#tqdm Globals # tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0 POSITION = 0
LEAVE = False LEAVE = False
# Pricing - Depends on the model https://openai.com/pricing # Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request # Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size. # If you are getting a MISMATCH LENGTH error, lower the batch size.
if 'gpt-3.5' in MODEL: if "gpt-3.5" in MODEL:
INPUTAPICOST = .002 INPUTAPICOST = 0.002
OUTPUTAPICOST = .002 OUTPUTAPICOST = 0.002
BATCHSIZE = 10 BATCHSIZE = 10
elif 'gpt-4' in MODEL: elif "gpt-4" in MODEL:
INPUTAPICOST = .01 INPUTAPICOST = 0.01
OUTPUTAPICOST = .03 OUTPUTAPICOST = 0.03
BATCHSIZE = 50 BATCHSIZE = 50
def handleAnim(filename, estimate): def handleAnim(filename, estimate):
global ESTIMATE global ESTIMATE
totalTokens = [0,0] totalTokens = [0, 0]
ESTIMATE = estimate ESTIMATE = estimate
if estimate: if estimate:
@ -76,17 +77,19 @@ def handleAnim(filename, estimate):
totalTokens[1] += translatedData[1][1] totalTokens[1] += translatedData[1][1]
# Print Total # Print Total
totalString = getResultString(['', totalTokens, None], end - start, 'TOTAL') totalString = getResultString(["", totalTokens, None], end - start, "TOTAL")
# Print any errors on maps # Print any errors on maps
if len(MISMATCH) > 0: if len(MISMATCH) > 0:
return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET return (
totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
)
else: else:
return totalString return totalString
else: else:
try: try:
with open('translated/' + filename, 'w', encoding='UTF-8') as outFile: with open("translated/" + filename, "w", encoding="UTF-8") as outFile:
start = time.time() start = time.time()
translatedData = openFiles(filename) translatedData = openFiles(filename)
@ -98,36 +101,50 @@ def handleAnim(filename, estimate):
totalTokens[0] += translatedData[1][0] totalTokens[0] += translatedData[1][0]
totalTokens[1] += translatedData[1][1] totalTokens[1] += translatedData[1][1]
except Exception: except Exception:
return 'Fail' return "Fail"
return getResultString(["", totalTokens, None], end - start, "TOTAL")
return getResultString(['', totalTokens, None], end - start, 'TOTAL')
def openFiles(filename): def openFiles(filename):
with open('files/' + filename, 'r', encoding='UTF-8-sig') as f: with open("files/" + filename, "r", encoding="UTF-8-sig") as f:
data = json.load(f) data = json.load(f)
# Map Files # Map Files
if '.json' in filename: if ".json" in filename:
translatedData = parseJSON(data, filename) translatedData = parseJSON(data, filename)
else: else:
raise NameError(filename + ' Not Supported') raise NameError(filename + " Not Supported")
return translatedData return translatedData
def getResultString(translatedData, translationTime, filename): def getResultString(translatedData, translationTime, filename):
# File Print String # File Print String
totalTokenstring =\ totalTokenstring = (
Fore.YELLOW +\ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
'[Input: ' + str(translatedData[1][0]) + ']'\ "[Output: "
'[Output: ' + str(translatedData[1][1]) + ']'\ + str(translatedData[1][1])
'[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\ + "]" "[Cost: ${:,.4f}".format(
(translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']' (translatedData[1][0] * 0.001 * INPUTAPICOST)
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
)
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None: if translatedData[2] == None:
# Success # Success
return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET return (
filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.GREEN
+ " \u2713 "
+ Fore.RESET
)
else: else:
# Fail # Fail
@ -136,20 +153,31 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
errorString = str(e) + Fore.RED errorString = str(e) + Fore.RED
return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\ return (
errorString + Fore.RESET filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.RED
+ " \u2717 "
+ errorString
+ Fore.RESET
)
def parseJSON(data, filename): def parseJSON(data, filename):
keys = list(data.keys()) keys = list(data.keys())
batches = [keys[i:i + BATCHSIZE] for i in range(0, len(keys), BATCHSIZE)] batches = [keys[i : i + BATCHSIZE] for i in range(0, len(keys), BATCHSIZE)]
totalTokens = [0, 0] totalTokens = [0, 0]
totalLines = 0 totalLines = 0
totalLines = len(batches) totalLines = len(batches)
global LOCK global LOCK
with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: with tqdm(
pbar.desc=filename bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE
pbar.total=totalLines ) as pbar:
pbar.desc = filename
pbar.total = totalLines
try: try:
result = translateJSON(batches, data, pbar) result = translateJSON(batches, data, pbar)
totalTokens[0] += result[0] totalTokens[0] += result[0]
@ -159,6 +187,7 @@ def parseJSON(data, filename):
return [data, totalTokens, e] return [data, totalTokens, e]
return [data, totalTokens, None] return [data, totalTokens, None]
def translateJSON(keys, data, pbar): def translateJSON(keys, data, pbar):
translatedBatch = [] translatedBatch = []
textHistory = [] textHistory = []
@ -172,20 +201,20 @@ def translateJSON(keys, data, pbar):
needTL = False needTL = False
for i in range(len(batch)): for i in range(len(batch)):
t = data[batch[i]] t = data[batch[i]]
if re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', t) or t == '': if re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", t) or t == "":
needTL = True needTL = True
if needTL is False and IGNORETLTEXT is True: if needTL is False and IGNORETLTEXT is True:
pbar.update(1) pbar.update(1)
continue continue
# Remove any textwrap and Furigana # Remove any textwrap and Furigana
for i in range(len(batch)): for i in range(len(batch)):
if FIXTEXTWRAP == True: if FIXTEXTWRAP == True:
# Textwrap # Textwrap
data[originalBatch[i]] = data[originalBatch[i]].replace('@b', ' ') data[originalBatch[i]] = data[originalBatch[i]].replace("@b", " ")
# Furigana # Furigana
rcodeMatch = re.findall(r'(@\[(.+?):.+?\])', batch[i]) rcodeMatch = re.findall(r"(@\[(.+?):.+?\])", batch[i])
if len(rcodeMatch) > 0: if len(rcodeMatch) > 0:
for match in rcodeMatch: for match in rcodeMatch:
batch[i] = batch[i].replace(match[0], match[1]) batch[i] = batch[i].replace(match[0], match[1])
@ -203,23 +232,22 @@ def translateJSON(keys, data, pbar):
# Format and Set Text # Format and Set Text
if len(batch) == len(translatedBatch): if len(batch) == len(translatedBatch):
for i in range(len(translatedBatch)): for i in range(len(translatedBatch)):
# Remove added speaker # Remove added speaker
translatedText = translatedBatch[i] translatedText = translatedBatch[i]
translatedText = re.sub(r'^.+?\s\|\s?', '', translatedText) translatedText = re.sub(r"^.+?\s\|\s?", "", translatedText)
# Textwrap # Textwrap
if '@n' in translatedText: if "@n" in translatedText:
match = re.search(r'.*@n(.*)', translatedText) match = re.search(r".*@n(.*)", translatedText)
if match != None: if match != None:
tlText = match.group(1) tlText = match.group(1)
tlText = textwrap.fill(tlText, width=WIDTH) tlText = textwrap.fill(tlText, width=WIDTH)
tlText = tlText.replace('\n', '@b') tlText = tlText.replace("\n", "@b")
translatedText = translatedText.replace(match.group(1), tlText) translatedText = translatedText.replace(match.group(1), tlText)
elif '@b' not in translatedText: elif "@b" not in translatedText:
translatedText = textwrap.fill(translatedText, width=WIDTH) translatedText = textwrap.fill(translatedText, width=WIDTH)
translatedText = translatedText.replace('\n', '@b') translatedText = translatedText.replace("\n", "@b")
# Set Data # Set Data
data[originalBatch[i]] = translatedText data[originalBatch[i]] = translatedText
@ -232,72 +260,74 @@ def translateJSON(keys, data, pbar):
continue continue
pbar.update(1) pbar.update(1)
return tokens return tokens
def subVars(jaString): def subVars(jaString):
jaString = jaString.replace('\u3000', ' ') jaString = jaString.replace("\u3000", " ")
# Nested # Nested
count = 0 count = 0
nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString) nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList) nestedList = set(nestedList)
if len(nestedList) != 0: if len(nestedList) != 0:
for icon in nestedList: for icon in nestedList:
jaString = jaString.replace(icon, '[Nested_' + str(count) + ']') jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString) iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList) iconList = set(iconList)
if len(iconList) != 0: if len(iconList) != 0:
for icon in iconList: for icon in iconList:
jaString = jaString.replace(icon, '[Ascii_' + str(count) + ']') jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString) colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList) colorList = set(colorList)
if len(colorList) != 0: if len(colorList) != 0:
for color in colorList: for color in colorList:
jaString = jaString.replace(color, '[Color_' + str(count) + ']') jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1 count += 1
# Names # Names
count = 0 count = 0
nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString) nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList) nameList = set(nameList)
if len(nameList) != 0: if len(nameList) != 0:
for name in nameList: for name in nameList:
jaString = jaString.replace(name, '[Noun_' + str(count) + ']') jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1 count += 1
# Variables # Variables
count = 0 count = 0
varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString) varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList) varList = set(varList)
if len(varList) != 0: if len(varList) != 0:
for var in varList: for var in varList:
jaString = jaString.replace(var, '[Var_' + str(count) + ']') jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
formatList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString) formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList) formatList = set(formatList)
if len(formatList) != 0: if len(formatList) != 0:
for var in formatList: for var in formatList:
jaString = jaString.replace(var, '[FCode_' + str(count) + ']') jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1 count += 1
# Put all lists in list and return # Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList] allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList] return [jaString, allList]
def resubVars(translatedText, allList): def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense # Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r'\[\s?.+?\s?\]', translatedText) matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0: if len(matchList) > 0:
for match in matchList: for match in matchList:
text = match.strip() text = match.strip()
@ -307,64 +337,70 @@ def resubVars(translatedText, allList):
count = 0 count = 0
if len(allList[0]) != 0: if len(allList[0]) != 0:
for var in allList[0]: for var in allList[0]:
translatedText = translatedText.replace('[Nested_' + str(count) + ']', var) translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
if len(allList[1]) != 0: if len(allList[1]) != 0:
for var in allList[1]: for var in allList[1]:
translatedText = translatedText.replace('[Ascii_' + str(count) + ']', var) translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
if len(allList[2]) != 0: if len(allList[2]) != 0:
for var in allList[2]: for var in allList[2]:
translatedText = translatedText.replace('[Color_' + str(count) + ']', var) translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1 count += 1
# Names # Names
count = 0 count = 0
if len(allList[3]) != 0: if len(allList[3]) != 0:
for var in allList[3]: for var in allList[3]:
translatedText = translatedText.replace('[Noun_' + str(count) + ']', var) translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1 count += 1
# Vars # Vars
count = 0 count = 0
if len(allList[4]) != 0: if len(allList[4]) != 0:
for var in allList[4]: for var in allList[4]:
translatedText = translatedText.replace('[Var_' + str(count) + ']', var) translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
if len(allList[5]) != 0: if len(allList[5]) != 0:
for var in allList[5]: for var in allList[5]:
translatedText = translatedText.replace('[FCode_' + str(count) + ']', var) translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1 count += 1
return translatedText return translatedText
def batchList(input_list, batch_size): def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0: if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer") raise ValueError("batch_size must be a positive integer")
return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)] return [
input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
]
def createContext(fullPromptFlag, subbedT): def createContext(fullPromptFlag, subbedT):
characters = 'Game Characters:\n\ characters = "Game Characters:\n\
達也 (Tatsuya) - Male\n\ 達也 (Tatsuya) - Male\n\
香織 (Kaori) - Female\n\ 香織 (Kaori) - Female\n\
岩瀬 (Iwase)\n\ 岩瀬 (Iwase)\n\
万蔵 (Manzou) - Male\n\ 万蔵 (Manzou) - Male\n\
結奈 (Yuuna) - Female\n\ 結奈 (Yuuna) - Female\n\
茅部 (Kayabe)\n\ 茅部 (Kayabe)\n\
' "
system = PROMPT + VOCAB if fullPromptFlag else \ system = (
f"\ PROMPT + VOCAB
if fullPromptFlag
else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\ - You translate everything, including erotic and explicit content.\n\
@ -376,9 +412,11 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\ {VOCAB}\n\
" "
user = f'{subbedT}' )
user = f"{subbedT}"
return characters, system, user return characters, system, user
def translateText(characters, system, user, history, penalty): def translateText(characters, system, user, history, penalty):
# Prompt # Prompt
msg = [{"role": "system", "content": system + characters}] msg = [{"role": "system", "content": system + characters}]
@ -391,9 +429,9 @@ def translateText(characters, system, user, history, penalty):
msg.extend([{"role": "system", "content": h} for h in history]) msg.extend([{"role": "system", "content": h} for h in history])
else: else:
msg.append({"role": "system", "content": history}) msg.append({"role": "system", "content": history})
# Content to TL # Content to TL
msg.append({"role": "user", "content": f'{user}'}) msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create( response = openai.chat.completions.create(
temperature=0, temperature=0,
frequency_penalty=penalty, frequency_penalty=penalty,
@ -402,18 +440,19 @@ def translateText(characters, system, user, history, penalty):
) )
return response return response
def cleanTranslatedText(translatedText, varResponse): def cleanTranslatedText(translatedText, varResponse):
placeholders = { placeholders = {
f'{LANGUAGE} Translation: ': '', f"{LANGUAGE} Translation: ": "",
'Translation: ': '', "Translation: ": "",
'っ': '', "っ": "",
'〜': '~', "〜": "~",
'ッ': '', "ッ": "",
'。': '.', "。": ".",
'Placeholder Text': '', "Placeholder Text": "",
'é' : 'e', "é": "e",
'—' : '-', "—": "-",
'ū' : 'u', "ū": "u",
# Add more replacements as needed # Add more replacements as needed
} }
for target, replacement in placeholders.items(): for target, replacement in placeholders.items():
@ -424,11 +463,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1]) translatedText = resubVars(translatedText, varResponse[1])
return translatedText return translatedText
def elongateCharacters(text): def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters # Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character # Using a positive lookbehind assertion to capture the preceding character
pattern = r'(?<=(.))ー+' pattern = r"(?<=(.))ー+"
# Define a replacement function that elongates the captured character # Define a replacement function that elongates the captured character
def repl(match): def repl(match):
char = match.group(1) # The character before the ー sequence char = match.group(1) # The character before the ー sequence
@ -438,8 +478,9 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text # Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text) return re.sub(pattern, repl, text)
def extractTranslation(translatedTextList, is_list): def extractTranslation(translatedTextList, is_list):
pattern = r'`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?' pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item. # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list: if is_list:
matchList = re.findall(pattern, translatedTextList) matchList = re.findall(pattern, translatedTextList)
@ -448,11 +489,12 @@ def extractTranslation(translatedTextList, is_list):
matchList = re.findall(pattern, translatedTextList) matchList = re.findall(pattern, translatedTextList)
return matchList[0][0] if matchList else translatedTextList return matchList[0][0] if matchList else translatedTextList
def countTokens(characters, system, user, history): def countTokens(characters, system, user, history):
inputTotalTokens = 0 inputTotalTokens = 0
outputTotalTokens = 0 outputTotalTokens = 0
enc = tiktoken.encoding_for_model('gpt-4') enc = tiktoken.encoding_for_model("gpt-4")
# Input # Input
if isinstance(history, list): if isinstance(history, list):
for line in history: for line in history:
@ -464,15 +506,17 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user)) inputTotalTokens += len(enc.encode(user))
# Output # Output
outputTotalTokens += round(len(enc.encode(user))*3) outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens] return [inputTotalTokens, outputTotalTokens]
def combineList(tlist, text): def combineList(tlist, text):
if isinstance(text, list): if isinstance(text, list):
return [t for sublist in tlist for t in sublist] return [t for sublist in tlist for t in sublist]
return tlist[0] return tlist[0]
@retry(exceptions=Exception, tries=5, delay=5) @retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag): def translateGPT(text, history, fullPromptFlag):
mismatch = False mismatch = False
@ -485,8 +529,12 @@ def translateGPT(text, history, fullPromptFlag):
for index, tItem in enumerate(tList): for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting # Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list): if isinstance(tItem, list):
payload = '\n'.join([f'`<Line{i}>{item}</Line{i}>`' for i, item in enumerate(tItem)]) payload = "\n".join(
payload = re.sub(r'(<Line\d+)(><)(\/Line\d+>)', r'\1>Placeholder Text<\3', payload) [f"`<Line{i}>{item}</Line{i}>`" for i, item in enumerate(tItem)]
)
payload = re.sub(
r"(<Line\d+)(><)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload
)
varResponse = subVars(payload) varResponse = subVars(payload)
subbedT = varResponse[0] subbedT = varResponse[0]
else: else:
@ -494,7 +542,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0] subbedT = varResponse[0]
# Things to Check before starting translation # Things to Check before starting translation
if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT): if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue continue
# Create Message # Create Message
@ -531,11 +579,13 @@ def translateGPT(text, history, fullPromptFlag):
extractedTranslations = extractTranslation(translatedText, True) extractedTranslations = extractTranslation(translatedText, True)
tList[index] = extractedTranslations tList[index] = extractedTranslations
if len(tItem) != len(extractedTranslations): if len(tItem) != len(extractedTranslations):
mismatch = True # Just here for breakpoint mismatch = True # Just here for breakpoint
# Create History # Create History
if not mismatch: if not mismatch:
history = extractedTranslations[-10:] # Update history if we have a list history = extractedTranslations[
-10:
] # Update history if we have a list
else: else:
history = text[-10:] history = text[-10:]
else: else:

255
modules/atelier.py
View file

@ -14,38 +14,41 @@ from tqdm import tqdm
# Open AI # Open AI
load_dotenv() load_dotenv()
if os.getenv('api').replace(' ', '') != '': if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv('api') openai.base_url = os.getenv("api")
openai.organization = os.getenv('org') openai.organization = os.getenv("org")
openai.api_key = os.getenv('key') openai.api_key = os.getenv("key")
#Globals # Globals
MODEL = os.getenv('model') MODEL = os.getenv("model")
TIMEOUT = int(os.getenv('timeout')) TIMEOUT = int(os.getenv("timeout"))
LANGUAGE=os.getenv('language').capitalize() LANGUAGE = os.getenv("language").capitalize()
INPUTAPICOST = .002 # Depends on the model https://openai.com/pricing INPUTAPICOST = 0.002 # Depends on the model https://openai.com/pricing
OUTPUTAPICOST = .002 OUTPUTAPICOST = 0.002
PROMPT = Path('prompt.txt').read_text(encoding='utf-8') PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path('vocab.txt').read_text(encoding='utf-8') VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv('threads')) # Controls how many threads are working on a single file (May have to drop this) THREADS = int(
os.getenv("threads")
) # Controls how many threads are working on a single file (May have to drop this)
LOCK = threading.Lock() LOCK = threading.Lock()
WIDTH = int(os.getenv('width')) WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv('listWidth')) LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 40 NOTEWIDTH = 40
MAXHISTORY = 10 MAXHISTORY = 10
ESTIMATE = '' ESTIMATE = ""
totalTokens = [0, 0] totalTokens = [0, 0]
NAMESLIST = [] NAMESLIST = []
#tqdm Globals # tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION=0 POSITION = 0
LEAVE=False LEAVE = False
# Translation Flags # Translation Flags
FIXTEXTWRAP = True FIXTEXTWRAP = True
IGNORETLTEXT = True IGNORETLTEXT = True
def handleAtelier(filename, estimate): def handleAtelier(filename, estimate):
global ESTIMATE, totalTokens global ESTIMATE, totalTokens
ESTIMATE = estimate ESTIMATE = estimate
@ -61,11 +64,11 @@ def handleAtelier(filename, estimate):
totalTokens[0] += translatedData[1][0] totalTokens[0] += translatedData[1][0]
totalTokens[1] += translatedData[1][1] totalTokens[1] += translatedData[1][1]
return getResultString(['', totalTokens, None], end - start, 'TOTAL') return getResultString(["", totalTokens, None], end - start, "TOTAL")
else: else:
try: try:
with open('translated/' + filename, 'w', encoding='utf-8') as outFile: with open("translated/" + filename, "w", encoding="utf-8") as outFile:
start = time.time() start = time.time()
translatedData = openFiles(filename) translatedData = openFiles(filename)
outFile.writelines(translatedData[0]) outFile.writelines(translatedData[0])
@ -77,29 +80,43 @@ def handleAtelier(filename, estimate):
totalTokens[0] += translatedData[1][0] totalTokens[0] += translatedData[1][0]
totalTokens[1] += translatedData[1][1] totalTokens[1] += translatedData[1][1]
except Exception: except Exception:
return 'Fail' return "Fail"
return getResultString(["", totalTokens, None], end - start, "TOTAL")
return getResultString(['', totalTokens, None], end - start, 'TOTAL')
def openFiles(filename): def openFiles(filename):
with open('files/' + filename, 'r', encoding='UTF-8') as f: with open("files/" + filename, "r", encoding="UTF-8") as f:
translatedData = parseText(f, filename) translatedData = parseText(f, filename)
return translatedData return translatedData
def getResultString(translatedData, translationTime, filename): def getResultString(translatedData, translationTime, filename):
# File Print String # File Print String
totalTokenstring =\ totalTokenstring = (
Fore.YELLOW +\ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
'[Input: ' + str(translatedData[1][0]) + ']'\ "[Output: "
'[Output: ' + str(translatedData[1][1]) + ']'\ + str(translatedData[1][1])
'[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\ + "]" "[Cost: ${:,.4f}".format(
(translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']' (translatedData[1][0] * 0.001 * INPUTAPICOST)
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
)
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] is None: if translatedData[2] is None:
# Success # Success
return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET return (
filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.GREEN
+ " \u2713 "
+ Fore.RESET
)
else: else:
# Fail # Fail
@ -107,9 +124,18 @@ def getResultString(translatedData, translationTime, filename):
raise translatedData[2] raise translatedData[2]
except Exception as e: except Exception as e:
errorString = str(e) + Fore.RED errorString = str(e) + Fore.RED
return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\ return (
errorString + Fore.RESET filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.RED
+ " \u2717 "
+ errorString
+ Fore.RESET
)
def parseText(data, filename): def parseText(data, filename):
totalLines = 0 totalLines = 0
global LOCK global LOCK
@ -117,10 +143,12 @@ def parseText(data, filename):
# Get total for progress bar # Get total for progress bar
linesList = data.readlines() linesList = data.readlines()
totalLines = len(linesList) totalLines = len(linesList)
with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: with tqdm(
pbar.desc=filename bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE
pbar.total=totalLines ) as pbar:
pbar.desc = filename
pbar.total = totalLines
try: try:
response = translateText(linesList, pbar) response = translateText(linesList, pbar)
except Exception as e: except Exception as e:
@ -128,32 +156,37 @@ def parseText(data, filename):
return [linesList, 0, e] return [linesList, 0, e]
return [response[0], response[1], None] return [response[0], response[1], None]
def translateText(data, pbar): def translateText(data, pbar):
textHistory = [] textHistory = []
maxHistory = MAXHISTORY maxHistory = MAXHISTORY
totalTokens = [0,0] totalTokens = [0, 0]
syncIndex = 0 syncIndex = 0
for i in range(len(data)): for i in range(len(data)):
if syncIndex > i: if syncIndex > i:
i = syncIndex i = syncIndex
match = re.findall(r'◆.+◆(.+)', data[i]) match = re.findall(r"◆.+◆(.+)", data[i])
if len(match) > 0: if len(match) > 0:
jaString = match[0] jaString = match[0]
### Translate ### Translate
# Remove any textwrap # Remove any textwrap
finalJAString = re.sub(r'\\n', ' ', jaString) finalJAString = re.sub(r"\\n", " ", jaString)
# Translate # Translate
response = translateGPT(finalJAString, 'Previous Text for Context: ' + ' '.join(textHistory), True) response = translateGPT(
finalJAString,
"Previous Text for Context: " + " ".join(textHistory),
True,
)
totalTokens[0] += response[1][0] totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1] totalTokens[1] += response[1][1]
translatedText = response[0] translatedText = response[0]
# TextHistory is what we use to give GPT Context, so thats appended here. # TextHistory is what we use to give GPT Context, so thats appended here.
textHistory.append('\"' + translatedText + '\"') textHistory.append('"' + translatedText + '"')
# Keep textHistory list at length maxHistory # Keep textHistory list at length maxHistory
if len(textHistory) > maxHistory: if len(textHistory) > maxHistory:
@ -161,81 +194,83 @@ def translateText(data, pbar):
# Textwrap # Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH) translatedText = textwrap.fill(translatedText, width=WIDTH)
translatedText = translatedText.replace('\n', '\\n') translatedText = translatedText.replace("\n", "\\n")
# Write # Write
data[i] = data[i].replace(match[0], translatedText) data[i] = data[i].replace(match[0], translatedText)
syncIndex = i + 1 syncIndex = i + 1
pbar.update() pbar.update()
return [data, totalTokens] return [data, totalTokens]
def subVars(jaString): def subVars(jaString):
jaString = jaString.replace('\u3000', ' ') jaString = jaString.replace("\u3000", " ")
# Nested # Nested
count = 0 count = 0
nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString) nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList) nestedList = set(nestedList)
if len(nestedList) != 0: if len(nestedList) != 0:
for icon in nestedList: for icon in nestedList:
jaString = jaString.replace(icon, '{Nested_' + str(count) + '}') jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString) iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList) iconList = set(iconList)
if len(iconList) != 0: if len(iconList) != 0:
for icon in iconList: for icon in iconList:
jaString = jaString.replace(icon, '{Ascii_' + str(count) + '}') jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString) colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList) colorList = set(colorList)
if len(colorList) != 0: if len(colorList) != 0:
for color in colorList: for color in colorList:
jaString = jaString.replace(color, '{Color_' + str(count) + '}') jaString = jaString.replace(color, "{Color_" + str(count) + "}")
count += 1 count += 1
# Names # Names
count = 0 count = 0
nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString) nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList) nameList = set(nameList)
if len(nameList) != 0: if len(nameList) != 0:
for name in nameList: for name in nameList:
jaString = jaString.replace(name, '{N_' + str(count) + '}') jaString = jaString.replace(name, "{N_" + str(count) + "}")
count += 1 count += 1
# Variables # Variables
count = 0 count = 0
varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString) varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList) varList = set(varList)
if len(varList) != 0: if len(varList) != 0:
for var in varList: for var in varList:
jaString = jaString.replace(var, '{Var_' + str(count) + '}') jaString = jaString.replace(var, "{Var_" + str(count) + "}")
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
if '笑えるよね.' in jaString: if "笑えるよね." in jaString:
print('t') print("t")
formatList = re.findall(r'[\\]+[\w]+\[.+?\]', jaString) formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
formatList = set(formatList) formatList = set(formatList)
if len(formatList) != 0: if len(formatList) != 0:
for var in formatList: for var in formatList:
jaString = jaString.replace(var, '{FCode_' + str(count) + '}') jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
count += 1 count += 1
# Put all lists in list and return # Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList] allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList] return [jaString, allList]
def resubVars(translatedText, allList): def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense # Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r'\[\s?.+?\s?\]', translatedText) matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0: if len(matchList) > 0:
for match in matchList: for match in matchList:
text = match.strip() text = match.strip()
@ -245,42 +280,42 @@ def resubVars(translatedText, allList):
count = 0 count = 0
if len(allList[0]) != 0: if len(allList[0]) != 0:
for var in allList[0]: for var in allList[0]:
translatedText = translatedText.replace('{Nested_' + str(count) + '}', var) translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
if len(allList[1]) != 0: if len(allList[1]) != 0:
for var in allList[1]: for var in allList[1]:
translatedText = translatedText.replace('{Ascii_' + str(count) + '}', var) translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
if len(allList[2]) != 0: if len(allList[2]) != 0:
for var in allList[2]: for var in allList[2]:
translatedText = translatedText.replace('{Color_' + str(count) + '}', var) translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
count += 1 count += 1
# Names # Names
count = 0 count = 0
if len(allList[3]) != 0: if len(allList[3]) != 0:
for var in allList[3]: for var in allList[3]:
translatedText = translatedText.replace('{N_' + str(count) + '}', var) translatedText = translatedText.replace("{N_" + str(count) + "}", var)
count += 1 count += 1
# Vars # Vars
count = 0 count = 0
if len(allList[4]) != 0: if len(allList[4]) != 0:
for var in allList[4]: for var in allList[4]:
translatedText = translatedText.replace('{Var_' + str(count) + '}', var) translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
if len(allList[5]) != 0: if len(allList[5]) != 0:
for var in allList[5]: for var in allList[5]:
translatedText = translatedText.replace('{FCode_' + str(count) + '}', var) translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
count += 1 count += 1
# Remove Color Variables Spaces # Remove Color Variables Spaces
@ -289,6 +324,7 @@ def resubVars(translatedText, allList):
# translatedText = re.sub(r'\s*(\\+c\[0+\])', r'\1', translatedText) # translatedText = re.sub(r'\s*(\\+c\[0+\])', r'\1', translatedText)
return translatedText return translatedText
@retry(exceptions=Exception, tries=5, delay=5) @retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(t, history, fullPromptFlag): def translateGPT(t, history, fullPromptFlag):
# Sub Vars # Sub Vars
@ -296,13 +332,13 @@ def translateGPT(t, history, fullPromptFlag):
subbedT = varResponse[0] subbedT = varResponse[0]
# If there isn't any Japanese in the text just skip # If there isn't any Japanese in the text just skip
if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]', subbedT): if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴ]+|[\uFF00-\uFFEF]", subbedT):
return(t, [0,0]) return (t, [0, 0])
# If ESTIMATE is True just count this as an execution and return. # If ESTIMATE is True just count this as an execution and return.
if ESTIMATE: if ESTIMATE:
enc = tiktoken.encoding_for_model('gpt-4') enc = tiktoken.encoding_for_model("gpt-4")
historyRaw = '' historyRaw = ""
if isinstance(history, list): if isinstance(history, list):
for line in history: for line in history:
historyRaw += line historyRaw += line
@ -310,26 +346,34 @@ def translateGPT(t, history, fullPromptFlag):
historyRaw = history historyRaw = history
inputTotalTokens = len(enc.encode(historyRaw)) + len(enc.encode(PROMPT)) inputTotalTokens = len(enc.encode(historyRaw)) + len(enc.encode(PROMPT))
outputTotalTokens = len(enc.encode(t)) * 2 # Estimating 2x the size of the original text outputTotalTokens = (
len(enc.encode(t)) * 2
) # Estimating 2x the size of the original text
totalTokens = [inputTotalTokens, outputTotalTokens] totalTokens = [inputTotalTokens, outputTotalTokens]
return (t, totalTokens) return (t, totalTokens)
# Characters # Characters
context = 'Game Characters:\ context = "Game Characters:\
Character: Surname:久高 Name:有史 == Surname:Kudaka Name:Yuushi - Gender: Male\ Character: Surname:久高 Name:有史 == Surname:Kudaka Name:Yuushi - Gender: Male\
Character: Surname:葛城 Name:碧璃 == Surname:Katsuragi Name:Midori - Gender: Female\ Character: Surname:葛城 Name:碧璃 == Surname:Katsuragi Name:Midori - Gender: Female\
Character: Surname:葛城 Name:依理子 == Surname:Katsuragi Name:Yoriko - Gender: Female\ Character: Surname:葛城 Name:依理子 == Surname:Katsuragi Name:Yoriko - Gender: Female\
Character: Surname:桐乃木 Name:奏 == Surname:Kirinogi Name:Kanade - Gender: Female\ Character: Surname:桐乃木 Name:奏 == Surname:Kirinogi Name:Kanade - Gender: Female\
Character: Surname:葛城 Name:光男 == Surname:Katsuragi Name:Mitsuo - Gender: Male\ Character: Surname:葛城 Name:光男 == Surname:Katsuragi Name:Mitsuo - Gender: Male\
Character: Surname:尾木 Name:優真 == Surname:Ogi Name:Yuuma - Gender: Male' Character: Surname:尾木 Name:優真 == Surname:Ogi Name:Yuuma - Gender: Male"
# Prompt # Prompt
if fullPromptFlag: if fullPromptFlag:
system = PROMPT system = PROMPT
user = 'Line to Translate = ' + subbedT user = "Line to Translate = " + subbedT
else: else:
system = 'Output ONLY the '+ LANGUAGE +' translation in the following format: `Translation: <'+ LANGUAGE.upper() +'_TRANSLATION>`' system = (
user = 'Line to Translate = ' + subbedT "Output ONLY the "
+ LANGUAGE
+ " translation in the following format: `Translation: <"
+ LANGUAGE.upper()
+ "_TRANSLATION>`"
)
user = "Line to Translate = " + subbedT
# Create Message List # Create Message List
msg = [] msg = []
@ -359,26 +403,29 @@ def translateGPT(t, history, fullPromptFlag):
translatedText = resubVars(translatedText, varResponse[1]) translatedText = resubVars(translatedText, varResponse[1])
# Remove Placeholder Text # Remove Placeholder Text
translatedText = translatedText.replace(LANGUAGE +' Translation: ', '') translatedText = translatedText.replace(LANGUAGE + " Translation: ", "")
translatedText = translatedText.replace('Translation: ', '') translatedText = translatedText.replace("Translation: ", "")
translatedText = translatedText.replace('Line to Translate = ', '') translatedText = translatedText.replace("Line to Translate = ", "")
translatedText = translatedText.replace('Translation = ', '') translatedText = translatedText.replace("Translation = ", "")
translatedText = translatedText.replace('Translate = ', '') translatedText = translatedText.replace("Translate = ", "")
translatedText = translatedText.replace(LANGUAGE +' Translation:', '') translatedText = translatedText.replace(LANGUAGE + " Translation:", "")
translatedText = translatedText.replace('Translation:', '') translatedText = translatedText.replace("Translation:", "")
translatedText = translatedText.replace('Line to Translate =', '') translatedText = translatedText.replace("Line to Translate =", "")
translatedText = translatedText.replace('Translation =', '') translatedText = translatedText.replace("Translation =", "")
translatedText = translatedText.replace('Translate =', '') translatedText = translatedText.replace("Translate =", "")
translatedText = translatedText.replace('っ', '') translatedText = translatedText.replace("っ", "")
translatedText = translatedText.replace('ッ', '') translatedText = translatedText.replace("ッ", "")
translatedText = translatedText.replace('ぁ', '') translatedText = translatedText.replace("ぁ", "")
translatedText = translatedText.replace('。', '.') translatedText = translatedText.replace("。", ".")
translatedText = translatedText.replace('、', ',') translatedText = translatedText.replace("、", ",")
translatedText = translatedText.replace('？', '?') translatedText = translatedText.replace("？", "?")
translatedText = translatedText.replace('！', '!') translatedText = translatedText.replace("！", "!")
# Return Translation # Return Translation
if len(translatedText) > 15 * len(t) or "I'm sorry, but I'm unable to assist with that translation" in translatedText: if (
len(translatedText) > 15 * len(t)
or "I'm sorry, but I'm unable to assist with that translation" in translatedText
):
raise Exception raise Exception
else: else:
return [translatedText, totalTokens] return [translatedText, totalTokens]

370
modules/csv.py
View file

@ -17,63 +17,66 @@ from tqdm import tqdm
# Open AI # Open AI
load_dotenv() load_dotenv()
if os.getenv('api').replace(' ', '') != '': if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv('api') openai.base_url = os.getenv("api")
openai.organization = os.getenv('org') openai.organization = os.getenv("org")
openai.api_key = os.getenv('key') openai.api_key = os.getenv("key")
#Globals # Globals
MODEL = os.getenv('model') MODEL = os.getenv("model")
TIMEOUT = int(os.getenv('timeout')) TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv('language').capitalize() LANGUAGE = os.getenv("language").capitalize()
PROMPT = Path('prompt.txt').read_text(encoding='utf-8') PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path('vocab.txt').read_text(encoding='utf-8') VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv('threads')) THREADS = int(os.getenv("threads"))
LOCK = threading.Lock() LOCK = threading.Lock()
WIDTH = int(os.getenv('width')) WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv('listWidth')) LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = int(os.getenv('noteWidth')) NOTEWIDTH = int(os.getenv("noteWidth"))
MAXHISTORY = 10 MAXHISTORY = 10
ESTIMATE = '' ESTIMATE = ""
TOKENS = [0, 0] TOKENS = [0, 0]
NAMESLIST = [] NAMESLIST = []
NAMES = False # Output a list of all the character names found NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = True # Overwrites textwrap FIXTEXTWRAP = True # Overwrites textwrap
IGNORETLTEXT = True # Ignores all translated text. IGNORETLTEXT = True # Ignores all translated text.
MISMATCH = [] # Lists files that thdata a mismatch error (Length of GPT list response is wrong) MISMATCH = [] # Lists files that thdata a mismatch error (Length of GPT list response is wrong)
BRACKETNAMES = False BRACKETNAMES = False
# Pricing - Depends on the model https://openai.com/pricing # Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request # Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size. # If you are getting a MISMATCH LENGTH error, lower the batch size.
if 'gpt-3.5' in MODEL: if "gpt-3.5" in MODEL:
INPUTAPICOST = .002 INPUTAPICOST = 0.002
OUTPUTAPICOST = .002 OUTPUTAPICOST = 0.002
BATCHSIZE = 10 BATCHSIZE = 10
FREQUENCY_PENALTY = 0.2 FREQUENCY_PENALTY = 0.2
elif 'gpt-4' in MODEL: elif "gpt-4" in MODEL:
INPUTAPICOST = .005 INPUTAPICOST = 0.005
OUTPUTAPICOST = .015 OUTPUTAPICOST = 0.015
BATCHSIZE = 20 BATCHSIZE = 20
FREQUENCY_PENALTY = 0.1 FREQUENCY_PENALTY = 0.1
#tqdm Globals # tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0 POSITION = 0
LEAVE = False LEAVE = False
PBAR = None PBAR = None
def handleCSV(filename, estimate): def handleCSV(filename, estimate):
global ESTIMATE, TOKENS global ESTIMATE, TOKENS
ESTIMATE = estimate ESTIMATE = estimate
if not ESTIMATE: if not ESTIMATE:
with open('translated/' + filename, 'w+t', newline='', encoding='utf-8-sig') as writeFile: with open(
"translated/" + filename, "w+t", newline="", encoding="utf-8-sig"
) as writeFile:
# Translate # Translate
start = time.time() start = time.time()
translatedData = openFiles(filename, writeFile) translatedData = openFiles(filename, writeFile)
# Print Result # Print Result
end = time.time() end = time.time()
tqdm.write(getResultString(translatedData, end - start, filename)) tqdm.write(getResultString(translatedData, end - start, filename))
@ -84,7 +87,7 @@ def handleCSV(filename, estimate):
# Translate # Translate
start = time.time() start = time.time()
translatedData = openFilesEstimate(filename) translatedData = openFilesEstimate(filename)
# Print Result # Print Result
end = time.time() end = time.time()
tqdm.write(getResultString(translatedData, end - start, filename)) tqdm.write(getResultString(translatedData, end - start, filename))
@ -92,41 +95,55 @@ def handleCSV(filename, estimate):
TOKENS[0] += translatedData[1][0] TOKENS[0] += translatedData[1][0]
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
# Print Total # Print Total
totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL') totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps # Print any errors on maps
if len(MISMATCH) > 0: if len(MISMATCH) > 0:
return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
else: else:
return totalString return totalString
def openFiles(filename, writeFile): def openFiles(filename, writeFile):
with open('files/' + filename, 'r', encoding='utf-8-sig') as readFile, writeFile: with open("files/" + filename, "r", encoding="utf-8-sig") as readFile, writeFile:
translatedData = parseCSV(readFile, writeFile, filename) translatedData = parseCSV(readFile, writeFile, filename)
return translatedData return translatedData
def openFilesEstimate(filename): def openFilesEstimate(filename):
with open('files/' + filename, 'r', encoding='utf-8-sig') as readFile: with open("files/" + filename, "r", encoding="utf-8-sig") as readFile:
translatedData = parseCSV(readFile, '', filename) translatedData = parseCSV(readFile, "", filename)
return translatedData return translatedData
def getResultString(translatedData, translationTime, filename): def getResultString(translatedData, translationTime, filename):
# File Print String # File Print String
totalTokenstring =\ totalTokenstring = (
Fore.YELLOW +\ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
'[Input: ' + str(translatedData[1][0]) + ']'\ "[Output: "
'[Output: ' + str(translatedData[1][1]) + ']'\ + str(translatedData[1][1])
'[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\ + "]" "[Cost: ${:,.4f}".format(
(translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']' (translatedData[1][0] * 0.001 * INPUTAPICOST)
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
)
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] is None: if translatedData[2] is None:
# Success # Success
return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET return (
filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.GREEN
+ " \u2713 "
+ Fore.RESET
)
else: else:
# Fail # Fail
try: try:
@ -134,38 +151,51 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
errorString = str(e) + Fore.RED errorString = str(e) + Fore.RED
return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\ return (
errorString + Fore.RESET filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.RED
+ " \u2717 "
+ errorString
+ Fore.RESET
)
def parseCSV(readFile, writeFile, filename): def parseCSV(readFile, writeFile, filename):
totalTokens = [0,0] totalTokens = [0, 0]
totalLines = 0 totalLines = 0
global LOCK global LOCK
format = '' format = ""
while format not in ['1', '2', '3']: while format not in ["1", "2", "3"]:
format = input('\n\nSelect the CSV Format:\n\n1. Translator++\n2. Single\n3. Multiple\n') format = input(
"\n\nSelect the CSV Format:\n\n1. Translator++\n2. Single\n3. Multiple\n"
)
match format: match format:
case '1': case "1":
format = '1' format = "1"
case '2': case "2":
format = '2' format = "2"
case '3': case "3":
format = '3' format = "3"
# Get total for progress bar # Get total for progress bar
totalLines = len(readFile.readlines()) totalLines = len(readFile.readlines())
readFile.seek(0) readFile.seek(0)
reader = csv.reader(readFile, delimiter=',') reader = csv.reader(readFile, delimiter=",")
if not ESTIMATE: if not ESTIMATE:
writer = csv.writer(writeFile, delimiter=',', quotechar='\"') writer = csv.writer(writeFile, delimiter=",", quotechar='"')
else: else:
writer = '' writer = ""
with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: with tqdm(
pbar.desc=filename bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE
pbar.total=totalLines ) as pbar:
pbar.desc = filename
pbar.total = totalLines
# Grab All Rows # Grab All Rows
data = [] data = []
@ -180,11 +210,12 @@ def parseCSV(readFile, writeFile, filename):
traceback.print_exc() traceback.print_exc()
return [data, totalTokens, None] return [data, totalTokens, None]
def translateCSV(data, pbar, writer, filename, translatedList, format): def translateCSV(data, pbar, writer, filename, translatedList, format):
global LOCK, ESTIMATE, PBAR global LOCK, ESTIMATE, PBAR
PBAR = pbar PBAR = pbar
translatedText = '' translatedText = ""
totalTokens = [0,0] totalTokens = [0, 0]
i = 0 i = 0
stringList = [] stringList = []
@ -193,7 +224,7 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
while i < len(data): while i < len(data):
match format: match format:
# T++ Format: Source Text on column 1. TL Target on Column 2 # T++ Format: Source Text on column 1. TL Target on Column 2
case '1': case "1":
# Get String # Get String
if i != 0: if i != 0:
if data[i][1] == "": if data[i][1] == "":
@ -202,7 +233,7 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
jaString = data[i][1] jaString = data[i][1]
# Remove Textwrap # Remove Textwrap
jaString = jaString.replace('\\n', ' ') jaString = jaString.replace("\\n", " ")
# Pass 1 # Pass 1
if not translatedList: if not translatedList:
@ -216,16 +247,16 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
# Add Wordwrap # Add Wordwrap
translatedText = textwrap.fill(translatedText, WIDTH) translatedText = textwrap.fill(translatedText, WIDTH)
translatedText = translatedText.replace('\n', '\\n') translatedText = translatedText.replace("\n", "\\n")
# Set Data # Set Data
data[i][1] = translatedText data[i][1] = translatedText
# Iterate # Iterate
i += 1 i += 1
# Target Format # Target Format
case '2': case "2":
# Set Values # Set Values
sourceColumn = 0 sourceColumn = 0
targetColumn = 1 targetColumn = 1
@ -234,7 +265,7 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
jaString = data[i][sourceColumn] jaString = data[i][sourceColumn]
# Remove Textwrap # Remove Textwrap
jaString = jaString.replace('\\n', ' ') jaString = jaString.replace("\\n", " ")
# Pass 1 # Pass 1
if not translatedList: if not translatedList:
@ -248,22 +279,22 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
# Add Wordwrap # Add Wordwrap
translatedText = textwrap.fill(translatedText, WIDTH) translatedText = textwrap.fill(translatedText, WIDTH)
translatedText = translatedText.replace('\n', '\\n') translatedText = translatedText.replace("\n", "\\n")
# Set Data # Set Data
data[i][targetColumn] = translatedText data[i][targetColumn] = translatedText
# Iterate # Iterate
i += 1 i += 1
# All Format # All Format
case '3': case "3":
# Set columns to translate. Leave empty to translate all. # Set columns to translate. Leave empty to translate all.
targetColumns = [] targetColumns = []
# False - Place translation in source column # False - Place translation in source column
# True - Place translation in next column # True - Place translation in next column
targetNextRow = True targetNextRow = True
for j in range(len(data[i])): for j in range(len(data[i])):
if j not in targetColumns: if j not in targetColumns:
@ -271,7 +302,7 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
jaString = data[i][j] jaString = data[i][j]
# Remove Textwrap # Remove Textwrap
jaString = jaString.replace('\\n', ' ') jaString = jaString.replace("\\n", " ")
# Pass 1 # Pass 1
if not translatedList: if not translatedList:
@ -285,14 +316,14 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
# Add Wordwrap # Add Wordwrap
translatedText = textwrap.fill(translatedText, WIDTH) translatedText = textwrap.fill(translatedText, WIDTH)
translatedText = translatedText.replace('\n', '\\n') translatedText = translatedText.replace("\n", "\\n")
# Set Data # Set Data
if targetNextRow: if targetNextRow:
data[i][j + 1] = translatedText data[i][j + 1] = translatedText
else: else:
data[i][j] = translatedText data[i][j] = translatedText
# Iterate # Iterate
i += 1 i += 1
@ -301,9 +332,9 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
# Set Progress # Set Progress
pbar.total = len(stringList) pbar.total = len(stringList)
pbar.refresh() pbar.refresh()
# Translate # Translate
response = translateGPT(stringList, '', True) response = translateGPT(stringList, "", True)
totalTokens[0] += response[1][0] totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1] totalTokens[1] += response[1][1]
translatedList = response[0] translatedList = response[0]
@ -333,27 +364,35 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
for row in data: for row in data:
writer.writerow(row) writer.writerow(row)
return totalTokens return totalTokens
return totalTokens return totalTokens
# Save some money and enter the character before translation # Save some money and enter the character before translation
def getSpeaker(speaker): def getSpeaker(speaker):
match speaker: match speaker:
case 'ファイン': case "ファイン":
return ['Fine', [0,0]] return ["Fine", [0, 0]]
case '': case "":
return ['', [0,0]] return ["", [0, 0]]
case _: case _:
# Store Speaker # Store Speaker
if speaker not in str(NAMESLIST): if speaker not in str(NAMESLIST):
response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False) response = translateGPT(
speaker,
"Reply with the " + LANGUAGE + " translation of the NPC name.",
False,
)
response[0] = response[0].title() response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s") response[0] = response[0].replace("'S", "'s")
# Retry if name doesn't translate for some reason # Retry if name doesn't translate for some reason
if re.search(r'([a-zA-Z？?])', response[0]) == None: if re.search(r"([a-zA-Z？?])", response[0]) == None:
response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False) response = translateGPT(
speaker,
"Reply with the " + LANGUAGE + " translation of the NPC name.",
False,
)
response[0] = response[0].title() response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s") response[0] = response[0].replace("'S", "'s")
@ -364,74 +403,76 @@ def getSpeaker(speaker):
else: else:
for i in range(len(NAMESLIST)): for i in range(len(NAMESLIST)):
if speaker == NAMESLIST[i][0]: if speaker == NAMESLIST[i][0]:
return [NAMESLIST[i][1],[0,0]] return [NAMESLIST[i][1], [0, 0]]
return [speaker,[0,0]] return [speaker, [0, 0]]
def subVars(jaString): def subVars(jaString):
jaString = jaString.replace('\u3000', ' ') jaString = jaString.replace("\u3000", " ")
# Nested # Nested
count = 0 count = 0
nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString) nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList) nestedList = set(nestedList)
if len(nestedList) != 0: if len(nestedList) != 0:
for icon in nestedList: for icon in nestedList:
jaString = jaString.replace(icon, '[Nested_' + str(count) + ']') jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString) iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList) iconList = set(iconList)
if len(iconList) != 0: if len(iconList) != 0:
for icon in iconList: for icon in iconList:
jaString = jaString.replace(icon, '[Ascii_' + str(count) + ']') jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
colorList = re.findall(r'([\\]+c\[\d+\][\\]+c|[\\]+c\[\d+\])', jaString) colorList = re.findall(r"([\\]+c\[\d+\][\\]+c|[\\]+c\[\d+\])", jaString)
colorList = set(colorList) colorList = set(colorList)
if len(colorList) != 0: if len(colorList) != 0:
for color in colorList: for color in colorList:
jaString = jaString.replace(color, '[Color_' + str(count) + ']') jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1 count += 1
# Names # Names
count = 0 count = 0
nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString) nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList) nameList = set(nameList)
if len(nameList) != 0: if len(nameList) != 0:
for name in nameList: for name in nameList:
jaString = jaString.replace(name, '[Noun_' + str(count) + ']') jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1 count += 1
# Variables # Variables
count = 0 count = 0
varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString) varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList) varList = set(varList)
if len(varList) != 0: if len(varList) != 0:
for var in varList: for var in varList:
jaString = jaString.replace(var, '[Var_' + str(count) + ']') jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
formatList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString) formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList) formatList = set(formatList)
if len(formatList) != 0: if len(formatList) != 0:
for var in formatList: for var in formatList:
jaString = jaString.replace(var, '[FCode_' + str(count) + ']') jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1 count += 1
# Put all lists in list and return # Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList] allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList] return [jaString, allList]
def resubVars(translatedText, allList): def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense # Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r'\[\s?.+?\s?\]', translatedText) matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0: if len(matchList) > 0:
for match in matchList: for match in matchList:
text = match.strip() text = match.strip()
@ -441,60 +482,66 @@ def resubVars(translatedText, allList):
count = 0 count = 0
if len(allList[0]) != 0: if len(allList[0]) != 0:
for var in allList[0]: for var in allList[0]:
translatedText = translatedText.replace('[Nested_' + str(count) + ']', var) translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
if len(allList[1]) != 0: if len(allList[1]) != 0:
for var in allList[1]: for var in allList[1]:
translatedText = translatedText.replace('[Ascii_' + str(count) + ']', var) translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
if len(allList[2]) != 0: if len(allList[2]) != 0:
for var in allList[2]: for var in allList[2]:
translatedText = translatedText.replace('[Color_' + str(count) + ']', var) translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1 count += 1
# Names # Names
count = 0 count = 0
if len(allList[3]) != 0: if len(allList[3]) != 0:
for var in allList[3]: for var in allList[3]:
translatedText = translatedText.replace('[Noun_' + str(count) + ']', var) translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1 count += 1
# Vars # Vars
count = 0 count = 0
if len(allList[4]) != 0: if len(allList[4]) != 0:
for var in allList[4]: for var in allList[4]:
translatedText = translatedText.replace('[Var_' + str(count) + ']', var) translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
if len(allList[5]) != 0: if len(allList[5]) != 0:
for var in allList[5]: for var in allList[5]:
translatedText = translatedText.replace('[FCode_' + str(count) + ']', var) translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1 count += 1
return translatedText return translatedText
def batchList(input_list, batch_size): def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0: if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer") raise ValueError("batch_size must be a positive integer")
return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)] return [
input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
]
def createContext(fullPromptFlag, subbedT): def createContext(fullPromptFlag, subbedT):
characters = 'Game Characters:\n\ characters = "Game Characters:\n\
クリスティーナ (Christina) - Female\n\ クリスティーナ (Christina) - Female\n\
リズ (Liz) - Female\n\ リズ (Liz) - Female\n\
' "
system = PROMPT + VOCAB if fullPromptFlag else \ system = (
f"\ PROMPT + VOCAB
if fullPromptFlag
else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\ - You translate everything, including erotic and explicit content.\n\
@ -506,12 +553,14 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\ {VOCAB}\n\
" "
)
if isinstance(subbedT, list): if isinstance(subbedT, list):
user = f'```json\n{subbedT}```' user = f"```json\n{subbedT}```"
else: else:
user = subbedT user = subbedT
return characters, system, user return characters, system, user
def translateText(characters, system, user, history, penalty, format): def translateText(characters, system, user, history, penalty, format):
# Prompt # Prompt
msg = [{"role": "system", "content": system + characters}] msg = [{"role": "system", "content": system + characters}]
@ -526,13 +575,13 @@ def translateText(characters, system, user, history, penalty, format):
msg.append({"role": "system", "content": history}) msg.append({"role": "system", "content": history})
# Response Format # Response Format
if format == 'json': if format == "json":
responseFormat = { "type": "json_object" } responseFormat = {"type": "json_object"}
else: else:
responseFormat = { "type": "text" } responseFormat = {"type": "text"}
# Content to TL # Content to TL
msg.append({"role": "user", "content": f'{user}'}) msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create( response = openai.chat.completions.create(
temperature=0, temperature=0,
frequency_penalty=penalty, frequency_penalty=penalty,
@ -542,18 +591,19 @@ def translateText(characters, system, user, history, penalty, format):
) )
return response return response
def cleanTranslatedText(translatedText, varResponse): def cleanTranslatedText(translatedText, varResponse):
placeholders = { placeholders = {
f'{LANGUAGE} Translation: ': '', f"{LANGUAGE} Translation: ": "",
'Translation: ': '', "Translation: ": "",
'っ': '', "っ": "",
'〜': '~', "〜": "~",
'ッ': '', "ッ": "",
'。': '.', "。": ".",
'「': '\\"', "「": '\\"',
'」': '\\"', "」": '\\"',
'- ': '-', "- ": "-",
'Placeholder Text': '', "Placeholder Text": "",
# Add more replacements as needed # Add more replacements as needed
} }
for target, replacement in placeholders.items(): for target, replacement in placeholders.items():
@ -564,11 +614,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1]) translatedText = resubVars(translatedText, varResponse[1])
return translatedText return translatedText
def elongateCharacters(text): def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters # Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character # Using a positive lookbehind assertion to capture the preceding character
pattern = r'(?<=(.))ー+' pattern = r"(?<=(.))ー+"
# Define a replacement function that elongates the captured character # Define a replacement function that elongates the captured character
def repl(match): def repl(match):
char = match.group(1) # The character before the ー sequence char = match.group(1) # The character before the ー sequence
@ -578,6 +629,7 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text # Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text) return re.sub(pattern, repl, text)
def extractTranslation(translatedTextList, is_list): def extractTranslation(translatedTextList, is_list):
try: try:
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList) translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
@ -590,15 +642,15 @@ def extractTranslation(translatedTextList, is_list):
return string_list[0] return string_list[0]
except Exception as e: except Exception as e:
PBAR.write(f'extractTranslation Error: {e} on String {translatedTextList}') PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
return None return None
def countTokens(characters, system, user, history): def countTokens(characters, system, user, history):
inputTotalTokens = 0 inputTotalTokens = 0
outputTotalTokens = 0 outputTotalTokens = 0
enc = tiktoken.encoding_for_model('gpt-4') enc = tiktoken.encoding_for_model("gpt-4")
# Input # Input
if isinstance(history, list): if isinstance(history, list):
for line in history: for line in history:
@ -610,26 +662,28 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user)) inputTotalTokens += len(enc.encode(user))
# Output # Output
outputTotalTokens += round(len(enc.encode(user))*3) outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens] return [inputTotalTokens, outputTotalTokens]
def combineList(tlist, text): def combineList(tlist, text):
if isinstance(text, list): if isinstance(text, list):
return [t for sublist in tlist for t in sublist] return [t for sublist in tlist for t in sublist]
return tlist[0] return tlist[0]
@retry(exceptions=Exception, tries=5, delay=5) @retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag): def translateGPT(text, history, fullPromptFlag):
global PBAR global PBAR
mismatch = False mismatch = False
totalTokens = [0, 0] totalTokens = [0, 0]
if isinstance(text, list): if isinstance(text, list):
format = 'json' format = "json"
tList = batchList(text, BATCHSIZE) tList = batchList(text, BATCHSIZE)
else: else:
format = 'text' format = "text"
tList = [text] tList = [text]
for index, tItem in enumerate(tList): for index, tItem in enumerate(tList):
@ -644,7 +698,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0] subbedT = varResponse[0]
# Things to Check before starting translation # Things to Check before starting translation
if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT): if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
if PBAR is not None: if PBAR is not None:
PBAR.update(len(tItem)) PBAR.update(len(tItem))
continue continue
@ -669,9 +723,13 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse) translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list): if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True) extractedTranslations = extractTranslation(translatedText, True)
if extractedTranslations == None or len(tItem) != len(extractedTranslations): if extractedTranslations == None or len(tItem) != len(
extractedTranslations
):
# Mismatch. Try Again # Mismatch. Try Again
response = translateText(characters, system, user, history, 0.05, format) response = translateText(
characters, system, user, history, 0.05, format
)
translatedText = response.choices[0].message.content translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens totalTokens[1] += response.usage.completion_tokens
@ -680,13 +738,17 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse) translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list): if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True) extractedTranslations = extractTranslation(translatedText, True)
if extractedTranslations == None or len(tItem) != len(extractedTranslations): if extractedTranslations == None or len(tItem) != len(
mismatch = True # Just here for breakpoint extractedTranslations
):
mismatch = True # Just here for breakpoint
# Set if no mismatch # Set if no mismatch
if mismatch == False: if mismatch == False:
tList[index] = extractedTranslations tList[index] = extractedTranslations
history = extractedTranslations[-10:] # Update history if we have a list history = extractedTranslations[
-10:
] # Update history if we have a list
else: else:
history = text[-10:] history = text[-10:]
mismatch = False mismatch = False
@ -697,7 +759,7 @@ def translateGPT(text, history, fullPromptFlag):
PBAR.update(len(tItem)) PBAR.update(len(tItem))
else: else:
# Ensure we're passing a single string to extractTranslation # Ensure we're passing a single string to extractTranslation
tList[index] = translatedText tList[index] = translatedText
finalList = combineList(tList, text) finalList = combineList(tList, text)
return [finalList, totalTokens] return [finalList, totalTokens]

471
modules/eushully.py
View file

@ -15,34 +15,34 @@ from tqdm import tqdm
# Open AI # Open AI
load_dotenv() load_dotenv()
if os.getenv('api').replace(' ', '') != '': if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv('api') openai.base_url = os.getenv("api")
openai.organization = os.getenv('org') openai.organization = os.getenv("org")
openai.api_key = os.getenv('key') openai.api_key = os.getenv("key")
#Globals # Globals
MODEL = os.getenv('model') MODEL = os.getenv("model")
TIMEOUT = int(os.getenv('timeout')) TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv('language').capitalize() LANGUAGE = os.getenv("language").capitalize()
PROMPT = Path('prompt.txt').read_text(encoding='utf-8') PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path('vocab.txt').read_text(encoding='utf-8') VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv('threads')) THREADS = int(os.getenv("threads"))
LOCK = threading.Lock() LOCK = threading.Lock()
WIDTH = int(os.getenv('width')) WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv('listWidth')) LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70 NOTEWIDTH = 70
MAXHISTORY = 10 MAXHISTORY = 10
ESTIMATE = '' ESTIMATE = ""
TOKENS = [0, 0] TOKENS = [0, 0]
NAMESLIST = [] NAMESLIST = []
NAMES = False # Output a list of all the character names found NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = True # Overwrites textwrap FIXTEXTWRAP = True # Overwrites textwrap
IGNORETLTEXT = False # Ignores all translated text. IGNORETLTEXT = False # Ignores all translated text.
MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
#tqdm Globals # tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0 POSITION = 0
LEAVE = False LEAVE = False
PBAR = None PBAR = None
@ -50,15 +50,16 @@ PBAR = None
# Pricing - Depends on the model https://openai.com/pricing # Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request # Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size. # If you are getting a MISMATCH LENGTH error, lower the batch size.
if 'gpt-3.5' in MODEL: if "gpt-3.5" in MODEL:
INPUTAPICOST = .002 INPUTAPICOST = 0.002
OUTPUTAPICOST = .002 OUTPUTAPICOST = 0.002
BATCHSIZE = 10 BATCHSIZE = 10
elif 'gpt-4' in MODEL: elif "gpt-4" in MODEL:
INPUTAPICOST = .005 INPUTAPICOST = 0.005
OUTPUTAPICOST = .015 OUTPUTAPICOST = 0.015
BATCHSIZE = 40 BATCHSIZE = 40
def handleEushully(filename, estimate): def handleEushully(filename, estimate):
global ESTIMATE global ESTIMATE
ESTIMATE = estimate ESTIMATE = estimate
@ -75,17 +76,21 @@ def handleEushully(filename, estimate):
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
# Print Total # Print Total
totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL') totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps # Print any errors on maps
if len(MISMATCH) > 0: if len(MISMATCH) > 0:
return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET return (
totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
)
else: else:
return totalString return totalString
else: else:
try: try:
with open('translated/' + filename, 'w', encoding='utf-8', errors='ignore') as outFile: with open(
"translated/" + filename, "w", encoding="utf-8", errors="ignore"
) as outFile:
start = time.time() start = time.time()
translatedData = openFiles(filename) translatedData = openFiles(filename)
@ -98,23 +103,36 @@ def handleEushully(filename, estimate):
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
except Exception: except Exception:
traceback.print_exc() traceback.print_exc()
return 'Fail' return "Fail"
return getResultString(["", TOKENS, None], end - start, "TOTAL")
return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def getResultString(translatedData, translationTime, filename): def getResultString(translatedData, translationTime, filename):
# File Print String # File Print String
totalTokenstring =\ totalTokenstring = (
Fore.YELLOW +\ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
'[Input: ' + str(translatedData[1][0]) + ']'\ "[Output: "
'[Output: ' + str(translatedData[1][1]) + ']'\ + str(translatedData[1][1])
'[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\ + "]" "[Cost: ${:,.4f}".format(
(translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']' (translatedData[1][0] * 0.001 * INPUTAPICOST)
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
)
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None: if translatedData[2] == None:
# Success # Success
return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET return (
filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.GREEN
+ " \u2713 "
+ Fore.RESET
)
else: else:
# Fail # Fail
@ -123,31 +141,41 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
errorString = str(e) + Fore.RED errorString = str(e) + Fore.RED
return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\ return (
errorString + Fore.RESET filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.RED
+ " \u2717 "
+ errorString
+ Fore.RESET
)
def openFiles(filename): def openFiles(filename):
with open('files/' + filename, 'r', encoding='utf-8') as readFile: with open("files/" + filename, "r", encoding="utf-8") as readFile:
translatedData = parseRegex(readFile, filename) translatedData = parseRegex(readFile, filename)
# Delete lines marked for deletion # Delete lines marked for deletion
finalData = [] finalData = []
for line in translatedData[0]: for line in translatedData[0]:
if line != '\\d\n': if line != "\\d\n":
finalData.append(line) finalData.append(line)
translatedData[0] = finalData translatedData[0] = finalData
return translatedData return translatedData
def parseRegex(readFile, filename): def parseRegex(readFile, filename):
totalTokens = [0,0] totalTokens = [0, 0]
# Read File into data # Read File into data
data = readFile.readlines() data = readFile.readlines()
# Create Progress Bar # Create Progress Bar
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
pbar.desc=filename pbar.desc = filename
try: try:
result = translateEushully(data, pbar, filename, []) result = translateEushully(data, pbar, filename, [])
@ -158,11 +186,12 @@ def parseRegex(readFile, filename):
return [data, totalTokens, e] return [data, totalTokens, e]
return [data, totalTokens, None] return [data, totalTokens, None]
def translateEushully(data, pbar, filename, translatedList): def translateEushully(data, pbar, filename, translatedList):
stringList = [] stringList = []
currentGroup = [] currentGroup = []
tokens = [0,0] tokens = [0, 0]
speaker = '' speaker = ""
voice = False voice = False
global LOCK, ESTIMATE, PBAR global LOCK, ESTIMATE, PBAR
i = 0 i = 0
@ -170,9 +199,9 @@ def translateEushully(data, pbar, filename, translatedList):
while i < len(data): while i < len(data):
voice = False voice = False
# Speaker # Speaker
if 'mov (global-int 46e2)' in data[i]: if "mov (global-int 46e2)" in data[i]:
# Get Speaker # Get Speaker
speaker = re.search(r'mov \(global-int 46e2\)\s(.+)', data[i]).group(1) speaker = re.search(r"mov \(global-int 46e2\)\s(.+)", data[i]).group(1)
response = getSpeaker(speaker) response = getSpeaker(speaker)
speaker = response[0] speaker = response[0]
tokens[0] += response[1][0] tokens[0] += response[1][0]
@ -180,32 +209,34 @@ def translateEushully(data, pbar, filename, translatedList):
i += 1 i += 1
# Show Text # Show Text
if any(x in data[i] for x in ['show-text']): if any(x in data[i] for x in ["show-text"]):
# Lines # Lines
regex = r'(.*?)"(.*)"' regex = r'(.*?)"(.*)"'
match = re.search(regex, data[i]) match = re.search(regex, data[i])
# Grab Strings # Grab Strings
if match != None and match.group(2) != '': if match != None and match.group(2) != "":
originalString = match.group(2) originalString = match.group(2)
jaString = match.group(2) jaString = match.group(2)
currentGroup = [jaString] currentGroup = [jaString]
while 'end-text-line' in data[i+1] and any(x in data[i+2] for x in ['show-text']): while "end-text-line" in data[i + 1] and any(
match = re.search(regex, data[i+2]) x in data[i + 2] for x in ["show-text"]
):
match = re.search(regex, data[i + 2])
if match != None: if match != None:
currentGroup.append(match.group(2)) currentGroup.append(match.group(2))
if translatedList == []: if translatedList == []:
del(data[i]) del data[i]
del(data[i]) del data[i]
jaString = ' '.join(currentGroup) jaString = " ".join(currentGroup)
# Pass 1 # Pass 1
if translatedList == []: if translatedList == []:
# Add String # Add String
if speaker: if speaker:
stringList.append(f'[{speaker}]: {jaString.strip()}') stringList.append(f"[{speaker}]: {jaString.strip()}")
else: else:
stringList.append(jaString.strip()) stringList.append(jaString.strip())
# Pass 2 # Pass 2
else: else:
# Get Text # Get Text
@ -222,51 +253,58 @@ def translateEushully(data, pbar, filename, translatedList):
translatedText = translatedText.replace('"', "'") translatedText = translatedText.replace('"', "'")
# Remove speaker # Remove speaker
if speaker != '': if speaker != "":
translatedText = re.sub(r'^\[?(.+?)\]?\s?[|:]\s?', '', translatedText) translatedText = re.sub(
r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText
)
# Textwrap # Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH) translatedText = textwrap.fill(translatedText, width=WIDTH)
translatedTextList = translatedText.split('\n') translatedTextList = translatedText.split("\n")
# Set Data # Set Data
if len(translatedTextList) > 1: if len(translatedTextList) > 1:
for j in range(len(translatedTextList)): for j in range(len(translatedTextList)):
if any(x in data[i] for x in ['show-text', 'set-string', 'concat']): if any(
del(data[i]) x in data[i]
data.insert(i, f'{match.group(1)}"{translatedTextList[j]}"\n') for x in ["show-text", "set-string", "concat"]
):
del data[i]
data.insert(
i, f'{match.group(1)}"{translatedTextList[j]}"\n'
)
i += 1 i += 1
if 'end-text-line' not in data[i]: if "end-text-line" not in data[i]:
data.insert(i, 'end-text-line 0\n') data.insert(i, "end-text-line 0\n")
i += 1 i += 1
else: else:
data[i] = f'{match.group(1)}"{translatedTextList[0]}"\n' data[i] = f'{match.group(1)}"{translatedTextList[0]}"\n'
speaker = '' speaker = ""
i += 1 i += 1
# Nothing relevant. Skip Line. # Nothing relevant. Skip Line.
else: else:
i += 1 i += 1
# Set String # Set String
elif 'set-string' in data[i]: elif "set-string" in data[i]:
# Lines # Lines
regex = r'(.*?)"(.*)"' regex = r'(.*?)"(.*)"'
match = re.search(regex, data[i]) match = re.search(regex, data[i])
# Grab Strings # Grab Strings
if match != None and match.group(2) != '': if match != None and match.group(2) != "":
originalString = match.group(2) originalString = match.group(2)
jaString = match.group(2) jaString = match.group(2)
currentGroup = [jaString] currentGroup = [jaString]
# Remove Textwrap # Remove Textwrap
jaString = jaString.replace('\\n', ' ') jaString = jaString.replace("\\n", " ")
# Pass 1 # Pass 1
if translatedList == []: if translatedList == []:
# Add String # Add String
stringList.append(jaString.strip()) stringList.append(jaString.strip())
# Pass 2 # Pass 2
else: else:
# Get Text # Get Text
@ -284,11 +322,11 @@ def translateEushully(data, pbar, filename, translatedList):
# Textwrap # Textwrap
translatedText = textwrap.fill(translatedText, width=LISTWIDTH) translatedText = textwrap.fill(translatedText, width=LISTWIDTH)
translatedText = translatedText.replace('\n', '\\n') translatedText = translatedText.replace("\n", "\\n")
# Set Data # Set Data
data[i] = data[i].replace(originalString, translatedText) data[i] = data[i].replace(originalString, translatedText)
speaker = '' speaker = ""
i += 1 i += 1
# Nothing relevant. Skip Line. # Nothing relevant. Skip Line.
@ -302,10 +340,10 @@ def translateEushully(data, pbar, filename, translatedList):
# Set Progress # Set Progress
pbar.total = len(stringList) pbar.total = len(stringList)
pbar.refresh() pbar.refresh()
# Translate # Translate
PBAR = pbar PBAR = pbar
response = translateGPT(stringList, '', True) response = translateGPT(stringList, "", True)
tokens[0] += response[1][0] tokens[0] += response[1][0]
tokens[1] += response[1][1] tokens[1] += response[1][1]
translatedList = response[0] translatedList = response[0]
@ -321,140 +359,143 @@ def translateEushully(data, pbar, filename, translatedList):
MISMATCH.append(filename) MISMATCH.append(filename)
return tokens return tokens
# Save some money and enter the character before translation # Save some money and enter the character before translation
def getSpeaker(speaker): def getSpeaker(speaker):
match speaker: match speaker:
case '1': case "1":
return ['Klaus', [0,0]] return ["Klaus", [0, 0]]
case '2': case "2":
return ['Helmina', [0,0]] return ["Helmina", [0, 0]]
case '3': case "3":
return ['Juliana', [0,0]] return ["Juliana", [0, 0]]
case '4': case "4":
return ['Reginia', [0,0]] return ["Reginia", [0, 0]]
case '5': case "5":
return ['Luciel', [0,0]] return ["Luciel", [0, 0]]
case '6': case "6":
return ['Mavislaine', [0,0]] return ["Mavislaine", [0, 0]]
case '7': case "7":
return ['Cerouge', [0,0]] return ["Cerouge", [0, 0]]
case '8': case "8":
return ['Maize', [0,0]] return ["Maize", [0, 0]]
case '9': case "9":
return ['Elvire', [0,0]] return ["Elvire", [0, 0]]
case 'a': case "a":
return ['Beatrice', [0,0]] return ["Beatrice", [0, 0]]
case '295': case "295":
return ['Orc', [0,0]] return ["Orc", [0, 0]]
case '232': case "232":
return ['Archangel', [0,0]] return ["Archangel", [0, 0]]
case '238': case "238":
return ['False Juliana', [0,0]] return ["False Juliana", [0, 0]]
case '239': case "239":
return ['False Regina', [0,0]] return ["False Regina", [0, 0]]
case '23a': case "23a":
return ['False Luciel', [0,0]] return ["False Luciel", [0, 0]]
case '23d': case "23d":
return ['False Mavislaine', [0,0]] return ["False Mavislaine", [0, 0]]
case 'cb': case "cb":
return ['Olga Niza Kite', [0,0]] return ["Olga Niza Kite", [0, 0]]
case 'c9': case "c9":
return ['Demon Beast Lupus', [0,0]] return ["Demon Beast Lupus", [0, 0]]
case 'ca': case "ca":
return ['Evelinael', [0,0]] return ["Evelinael", [0, 0]]
case '10': case "10":
return ['Eukleia', [0,0]] return ["Eukleia", [0, 0]]
case '15': case "15":
return ['Lily', [0,0]] return ["Lily", [0, 0]]
case '16': case "16":
return ['Kupuko', [0,0]] return ["Kupuko", [0, 0]]
case 'b': case "b":
return ['Ramiel', [0,0]] return ["Ramiel", [0, 0]]
case 'c': case "c":
return ['Henriette', [0,0]] return ["Henriette", [0, 0]]
case 'd': case "d":
return ['Camilla', [0,0]] return ["Camilla", [0, 0]]
case 'cc': case "cc":
return ['Gogonaua', [0,0]] return ["Gogonaua", [0, 0]]
case '65': case "65":
return ['Demon Lord Reyvalois', [0,0]] return ["Demon Lord Reyvalois", [0, 0]]
case 'd0': case "d0":
return ['Demon Ranwald', [0,0]] return ["Demon Ranwald", [0, 0]]
case '205': case "205":
return ['Vanqueor', [0,0]] return ["Vanqueor", [0, 0]]
case '66': case "66":
return ['Angel Martina', [0,0]] return ["Angel Martina", [0, 0]]
case '21f': case "21f":
return ['Hiten Demon', [0,0]] return ["Hiten Demon", [0, 0]]
case 'd2': case "d2":
return ['Lena Eli', [0,0]] return ["Lena Eli", [0, 0]]
case _: case _:
return ['Unknown', [0,0]] return ["Unknown", [0, 0]]
def subVars(jaString): def subVars(jaString):
jaString = jaString.replace('\u3000', ' ') jaString = jaString.replace("\u3000", " ")
# Nested # Nested
count = 0 count = 0
nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString) nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList) nestedList = set(nestedList)
if len(nestedList) != 0: if len(nestedList) != 0:
for icon in nestedList: for icon in nestedList:
jaString = jaString.replace(icon, '[Nested_' + str(count) + ']') jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString) iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList) iconList = set(iconList)
if len(iconList) != 0: if len(iconList) != 0:
for icon in iconList: for icon in iconList:
jaString = jaString.replace(icon, '[Ascii_' + str(count) + ']') jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString) colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList) colorList = set(colorList)
if len(colorList) != 0: if len(colorList) != 0:
for color in colorList: for color in colorList:
jaString = jaString.replace(color, '[Color_' + str(count) + ']') jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1 count += 1
# Names # Names
count = 0 count = 0
nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString) nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList) nameList = set(nameList)
if len(nameList) != 0: if len(nameList) != 0:
for name in nameList: for name in nameList:
jaString = jaString.replace(name, '[Noun_' + str(count) + ']') jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1 count += 1
# Variables # Variables
count = 0 count = 0
varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString) varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList) varList = set(varList)
if len(varList) != 0: if len(varList) != 0:
for var in varList: for var in varList:
jaString = jaString.replace(var, '[Var_' + str(count) + ']') jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
formatList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString) formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList) formatList = set(formatList)
if len(formatList) != 0: if len(formatList) != 0:
for var in formatList: for var in formatList:
jaString = jaString.replace(var, '[FCode_' + str(count) + ']') jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1 count += 1
# Put all lists in list and return # Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList] allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList] return [jaString, allList]
def resubVars(translatedText, allList): def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense # Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r'\[\s?.+?\s?\]', translatedText) matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0: if len(matchList) > 0:
for match in matchList: for match in matchList:
text = match.strip() text = match.strip()
@ -464,59 +505,65 @@ def resubVars(translatedText, allList):
count = 0 count = 0
if len(allList[0]) != 0: if len(allList[0]) != 0:
for var in allList[0]: for var in allList[0]:
translatedText = translatedText.replace('[Nested_' + str(count) + ']', var) translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
if len(allList[1]) != 0: if len(allList[1]) != 0:
for var in allList[1]: for var in allList[1]:
translatedText = translatedText.replace('[Ascii_' + str(count) + ']', var) translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
if len(allList[2]) != 0: if len(allList[2]) != 0:
for var in allList[2]: for var in allList[2]:
translatedText = translatedText.replace('[Color_' + str(count) + ']', var) translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1 count += 1
# Names # Names
count = 0 count = 0
if len(allList[3]) != 0: if len(allList[3]) != 0:
for var in allList[3]: for var in allList[3]:
translatedText = translatedText.replace('[Noun_' + str(count) + ']', var) translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1 count += 1
# Vars # Vars
count = 0 count = 0
if len(allList[4]) != 0: if len(allList[4]) != 0:
for var in allList[4]: for var in allList[4]:
translatedText = translatedText.replace('[Var_' + str(count) + ']', var) translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
if len(allList[5]) != 0: if len(allList[5]) != 0:
for var in allList[5]: for var in allList[5]:
translatedText = translatedText.replace('[FCode_' + str(count) + ']', var) translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1 count += 1
return translatedText return translatedText
def batchList(input_list, batch_size): def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0: if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer") raise ValueError("batch_size must be a positive integer")
return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)] return [
input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
]
def createContext(fullPromptFlag, subbedT): def createContext(fullPromptFlag, subbedT):
characters = 'Game Characters:\n\ characters = "Game Characters:\n\
グレイス (Grace) - Female\n\ グレイス (Grace) - Female\n\
' "
system = PROMPT + VOCAB if fullPromptFlag else \ system = (
f"\ PROMPT + VOCAB
if fullPromptFlag
else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\ - You translate everything, including erotic and explicit content.\n\
@ -528,9 +575,11 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\ {VOCAB}\n\
" "
user = f'{subbedT}' )
user = f"{subbedT}"
return characters, system, user return characters, system, user
def translateText(characters, system, user, history, penalty): def translateText(characters, system, user, history, penalty):
# Prompt # Prompt
msg = [{"role": "system", "content": system + characters}] msg = [{"role": "system", "content": system + characters}]
@ -543,9 +592,9 @@ def translateText(characters, system, user, history, penalty):
msg.extend([{"role": "system", "content": h} for h in history]) msg.extend([{"role": "system", "content": h} for h in history])
else: else:
msg.append({"role": "system", "content": history}) msg.append({"role": "system", "content": history})
# Content to TL # Content to TL
msg.append({"role": "user", "content": f'{user}'}) msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create( response = openai.chat.completions.create(
temperature=0, temperature=0,
frequency_penalty=penalty, frequency_penalty=penalty,
@ -554,23 +603,24 @@ def translateText(characters, system, user, history, penalty):
) )
return response return response
def cleanTranslatedText(translatedText, varResponse): def cleanTranslatedText(translatedText, varResponse):
placeholders = { placeholders = {
f'{LANGUAGE} Translation: ': '', f"{LANGUAGE} Translation: ": "",
'Translation: ': '', "Translation: ": "",
'っ': '', "っ": "",
'〜': '~', "〜": "~",
'ッ': '', "ッ": "",
'。': '.', "。": ".",
'< ': '<', "< ": "<",
'</ ': '</', "</ ": "</",
' >': '>', " >": ">",
'「': '\"', "「": '"',
'」': '\"', "」": '"',
'Placeholder Text': '', "Placeholder Text": "",
'- chan': '-chan', "- chan": "-chan",
'- kun': '-kun', "- kun": "-kun",
'- san': '-san', "- san": "-san",
# Add more replacements as needed # Add more replacements as needed
} }
for target, replacement in placeholders.items(): for target, replacement in placeholders.items():
@ -581,11 +631,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1]) translatedText = resubVars(translatedText, varResponse[1])
return translatedText return translatedText
def elongateCharacters(text): def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters # Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character # Using a positive lookbehind assertion to capture the preceding character
pattern = r'(?<=(.))ー+' pattern = r"(?<=(.))ー+"
# Define a replacement function that elongates the captured character # Define a replacement function that elongates the captured character
def repl(match): def repl(match):
char = match.group(1) # The character before the ー sequence char = match.group(1) # The character before the ー sequence
@ -595,8 +646,9 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text # Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text) return re.sub(pattern, repl, text)
def extractTranslation(translatedTextList, is_list): def extractTranslation(translatedTextList, is_list):
pattern = r'`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?' pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item. # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list: if is_list:
matchList = re.findall(pattern, translatedTextList) matchList = re.findall(pattern, translatedTextList)
@ -605,11 +657,12 @@ def extractTranslation(translatedTextList, is_list):
matchList = re.findall(pattern, translatedTextList) matchList = re.findall(pattern, translatedTextList)
return matchList[0][0] if matchList else translatedTextList return matchList[0][0] if matchList else translatedTextList
def countTokens(characters, system, user, history): def countTokens(characters, system, user, history):
inputTotalTokens = 0 inputTotalTokens = 0
outputTotalTokens = 0 outputTotalTokens = 0
enc = tiktoken.encoding_for_model('gpt-4') enc = tiktoken.encoding_for_model("gpt-4")
# Input # Input
if isinstance(history, list): if isinstance(history, list):
for line in history: for line in history:
@ -621,19 +674,21 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user)) inputTotalTokens += len(enc.encode(user))
# Output # Output
outputTotalTokens += round(len(enc.encode(user))*3) outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens] return [inputTotalTokens, outputTotalTokens]
def combineList(tlist, text): def combineList(tlist, text):
if isinstance(text, list): if isinstance(text, list):
return [t for sublist in tlist for t in sublist] return [t for sublist in tlist for t in sublist]
return tlist[0] return tlist[0]
@retry(exceptions=Exception, tries=5, delay=5) @retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag): def translateGPT(text, history, fullPromptFlag):
global PBAR global PBAR
mismatch = False mismatch = False
totalTokens = [0, 0] totalTokens = [0, 0]
if isinstance(text, list): if isinstance(text, list):
@ -644,8 +699,12 @@ def translateGPT(text, history, fullPromptFlag):
for index, tItem in enumerate(tList): for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting # Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list): if isinstance(tItem, list):
payload = '\n'.join([f'`<Line{i}>{item}</Line{i}>`' for i, item in enumerate(tItem)]) payload = "\n".join(
payload = re.sub(r'(<Line\d+)(><)(\/Line\d+>)', r'\1>Placeholder Text<\3', payload) [f"`<Line{i}>{item}</Line{i}>`" for i, item in enumerate(tItem)]
)
payload = re.sub(
r"(<Line\d+)(><)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload
)
varResponse = subVars(payload) varResponse = subVars(payload)
subbedT = varResponse[0] subbedT = varResponse[0]
else: else:
@ -653,7 +712,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0] subbedT = varResponse[0]
# Things to Check before starting translation # Things to Check before starting translation
if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT): if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
if PBAR is not None: if PBAR is not None:
PBAR.update(len(tItem)) PBAR.update(len(tItem))
continue continue
@ -692,14 +751,16 @@ def translateGPT(text, history, fullPromptFlag):
extractedTranslations = extractTranslation(translatedText, True) extractedTranslations = extractTranslation(translatedText, True)
tList[index] = extractedTranslations tList[index] = extractedTranslations
if len(tItem) != len(extractedTranslations): if len(tItem) != len(extractedTranslations):
mismatch = True # Just here for breakpoint mismatch = True # Just here for breakpoint
# Create History # Create History
with LOCK: with LOCK:
if PBAR is not None: if PBAR is not None:
PBAR.update(len(tItem)) PBAR.update(len(tItem))
if not mismatch: if not mismatch:
history = extractedTranslations[-10:] # Update history if we have a list history = extractedTranslations[
-10:
] # Update history if we have a list
else: else:
history = text[-10:] history = text[-10:]
else: else:

333
modules/images.py
View file

@ -14,72 +14,77 @@ from dotenv import load_dotenv
from retry import retry from retry import retry
from tqdm import tqdm from tqdm import tqdm
#Globals # Globals
MODEL = os.getenv('model') MODEL = os.getenv("model")
TIMEOUT = int(os.getenv('timeout')) TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv('language').capitalize() LANGUAGE = os.getenv("language").capitalize()
PROMPT = Path('prompt.txt').read_text(encoding='utf-8') PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path('vocab.txt').read_text(encoding='utf-8') VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv('threads')) THREADS = int(os.getenv("threads"))
LOCK = threading.Lock() LOCK = threading.Lock()
PBAR = None PBAR = None
WIDTH = int(os.getenv('width')) WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv('listWidth')) LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = int(os.getenv('noteWidth')) NOTEWIDTH = int(os.getenv("noteWidth"))
MAXHISTORY = 10 MAXHISTORY = 10
ESTIMATE = '' ESTIMATE = ""
TOKENS = [0, 0] TOKENS = [0, 0]
NAMESLIST = [] NAMESLIST = []
MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
# Open AI # Open AI
load_dotenv() load_dotenv()
if os.getenv('api').replace(' ', '') != '': if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv('api') openai.base_url = os.getenv("api")
openai.organization = os.getenv('org') openai.organization = os.getenv("org")
openai.api_key = os.getenv('key') openai.api_key = os.getenv("key")
# Pricing - Depends on the model https://openai.com/pricing # Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request # Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size. # If you are getting a MISMATCH LENGTH error, lower the batch size.
if 'gpt-3.5' in MODEL: if "gpt-3.5" in MODEL:
INPUTAPICOST = .002 INPUTAPICOST = 0.002
OUTPUTAPICOST = .002 OUTPUTAPICOST = 0.002
BATCHSIZE = 10 BATCHSIZE = 10
FREQUENCY_PENALTY = 0.2 FREQUENCY_PENALTY = 0.2
elif 'gpt-4' in MODEL: elif "gpt-4" in MODEL:
INPUTAPICOST = .005 INPUTAPICOST = 0.005
OUTPUTAPICOST = .015 OUTPUTAPICOST = 0.015
BATCHSIZE = 20 BATCHSIZE = 20
FREQUENCY_PENALTY = 0.1 FREQUENCY_PENALTY = 0.1
#tqdm Globals # tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0 POSITION = 0
LEAVE = False LEAVE = False
def handleImages(folderName, estimate): def handleImages(folderName, estimate):
global ESTIMATE, TOKENS global ESTIMATE, TOKENS
ESTIMATE = estimate ESTIMATE = estimate
start = time.time() start = time.time()
# Translate Strings # Translate Strings
translatedData = openFiles(f'files/{folderName}') translatedData = openFiles(f"files/{folderName}")
# Write Strings to Images # Write Strings to Images
if not ESTIMATE: if not ESTIMATE:
if not os.path.exists(f'translated/{folderName}'): if not os.path.exists(f"translated/{folderName}"):
os.mkdir(f'translated/{folderName}') os.mkdir(f"translated/{folderName}")
for i in range(len(translatedData[0][0])): for i in range(len(translatedData[0][0])):
try: try:
translatedList = translatedData[0][0] translatedList = translatedData[0][0]
originalList = translatedData[0][1] originalList = translatedData[0][1]
dimensionsList = translatedData[0][2] dimensionsList = translatedData[0][2]
image = stringToImage(translatedList[i], dimensionsList[i][0], dimensionsList[i][1]) image = stringToImage(
image.save(rf'translated/{folderName}/{translatedList[i]}.png', quality=100) translatedList[i], dimensionsList[i][0], dimensionsList[i][1]
)
image.save(
rf"translated/{folderName}/{translatedList[i]}.png", quality=100
)
except Exception as e: except Exception as e:
PBAR.write(f'{translatedList[i]}: {str(e)}') PBAR.write(f"{translatedList[i]}: {str(e)}")
#Ignore Error # Ignore Error
# Print File # Print File
end = time.time() end = time.time()
@ -89,43 +94,67 @@ def handleImages(folderName, estimate):
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
# Print Total # Print Total
totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL') totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps # Print any errors on maps
if len(MISMATCH) > 0: if len(MISMATCH) > 0:
return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET return totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
else: else:
return totalString return totalString
def openFiles(folderName): def openFiles(folderName):
global PBAR global PBAR
if os.path.isdir(folderName): if os.path.isdir(folderName):
imageList = [[],[]] imageList = [[], []]
imageList = processImagesDir(folderName, imageList) imageList = processImagesDir(folderName, imageList)
# Start Translation # Start Translation
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=folderName, total=len(imageList[0])) as PBAR: with tqdm(
bar_format=BAR_FORMAT,
position=POSITION,
leave=LEAVE,
desc=folderName,
total=len(imageList[0]),
) as PBAR:
translatedData = translateImages(imageList) translatedData = translateImages(imageList)
translatedData = [[translatedData[0], imageList[0], imageList[1]], translatedData[1], translatedData[2]] translatedData = [
[translatedData[0], imageList[0], imageList[1]],
translatedData[1],
translatedData[2],
]
return translatedData return translatedData
else: else:
print("The provided directory path does not exist.") print("The provided directory path does not exist.")
def getResultString(translatedData, translationTime, filename): def getResultString(translatedData, translationTime, filename):
# File Print String # File Print String
totalTokenstring =\ totalTokenstring = (
Fore.YELLOW +\ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
'[Input: ' + str(translatedData[1][0]) + ']'\ "[Output: "
'[Output: ' + str(translatedData[1][1]) + ']'\ + str(translatedData[1][1])
'[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\ + "]" "[Cost: ${:,.4f}".format(
(translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']' (translatedData[1][0] * 0.001 * INPUTAPICOST)
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
)
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] is None: if translatedData[2] is None:
# Success # Success
return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET return (
filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.GREEN
+ " \u2713 "
+ Fore.RESET
)
else: else:
# Fail # Fail
try: try:
@ -133,8 +162,17 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
errorString = str(e) + Fore.RED errorString = str(e) + Fore.RED
return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\ return (
errorString + Fore.RESET filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.RED
+ " \u2717 "
+ errorString
+ Fore.RESET
)
def getFontSize(text, image_width, image_height, font_path): def getFontSize(text, image_width, image_height, font_path):
# Start with a high font size and keep reducing it until the text fits within the image bounds # Start with a high font size and keep reducing it until the text fits within the image bounds
@ -142,50 +180,59 @@ def getFontSize(text, image_width, image_height, font_path):
while font_size > 0: while font_size > 0:
font = ImageFont.truetype(font_path, font_size) font = ImageFont.truetype(font_path, font_size)
text_bbox = ImageDraw.Draw(Image.new('RGB', (1, 1))).textbbox((0, 0), text, font=font) text_bbox = ImageDraw.Draw(Image.new("RGB", (1, 1))).textbbox(
(0, 0), text, font=font
)
text_width = text_bbox[2] - text_bbox[0] text_width = text_bbox[2] - text_bbox[0]
text_height = text_bbox[3] - text_bbox[1] + 5 text_height = text_bbox[3] - text_bbox[1] + 5
if text_width <= image_width and text_height <= image_height: if text_width <= image_width and text_height <= image_height:
return font_size return font_size
font_size -= 1 font_size -= 1
return font_size return font_size
def stringToImage(text, width, height, font_path='fonts/TsunagiGothic.ttf', scale_factor=4):
def stringToImage(
text, width, height, font_path="fonts/TsunagiGothic.ttf", scale_factor=4
):
# Increase the resolution # Increase the resolution
scaled_width = int(width * scale_factor) scaled_width = int(width * scale_factor)
scaled_height = int(height * scale_factor) scaled_height = int(height * scale_factor)
# Find the appropriate font size for the scaled up image # Find the appropriate font size for the scaled up image
font_size = getFontSize(text, scaled_width, scaled_height, font_path) font_size = getFontSize(text, scaled_width, scaled_height, font_path)
if font_size == 0: if font_size == 0:
raise ValueError("Text is too long to fit in the supplied dimensions.") raise ValueError("Text is too long to fit in the supplied dimensions.")
# Create a new image with the scaled width and height and a transparent background # Create a new image with the scaled width and height and a transparent background
image = Image.new('RGBA', (scaled_width, scaled_height), (255, 255, 255, 0)) image = Image.new("RGBA", (scaled_width, scaled_height), (255, 255, 255, 0))
# Create a drawing context # Create a drawing context
draw = ImageDraw.Draw(image) draw = ImageDraw.Draw(image)
# Load the appropriate font # Load the appropriate font
font = ImageFont.truetype(font_path, font_size) font = ImageFont.truetype(font_path, font_size)
# Calculate the size of the text to center it # Calculate the size of the text to center it
text_bbox = draw.textbbox((0, 0), text, font=font) text_bbox = draw.textbbox((0, 0), text, font=font)
text_width = text_bbox[2] - text_bbox[0] text_width = text_bbox[2] - text_bbox[0]
text_height = text_bbox[3] - text_bbox[1] text_height = text_bbox[3] - text_bbox[1]
x = (scaled_width - text_width) // 2 x = (scaled_width - text_width) // 2
y = (scaled_height - text_height) // 2 y = (scaled_height - text_height) // 2
# Draw the text on the image # Draw the text on the image
draw.text((x, y), text, font=font, fill=(255, 255, 255, 255)) draw.text((x, y), text, font=font, fill=(255, 255, 255, 255))
# Resize back to the original dimensions to get a clearer text rendering # Resize back to the original dimensions to get a clearer text rendering
image = image.resize((width, height), Image.LANCZOS,) image = image.resize(
(width, height),
Image.LANCZOS,
)
return image return image
def getImageDimensions(file_path): def getImageDimensions(file_path):
try: try:
with Image.open(file_path) as img: with Image.open(file_path) as img:
@ -195,10 +242,11 @@ def getImageDimensions(file_path):
print(f"Error reading {file_path}: {e}") print(f"Error reading {file_path}: {e}")
return None, None return None, None
def processImagesDir(directory_path, imageList): def processImagesDir(directory_path, imageList):
for file_name in os.listdir(directory_path): for file_name in os.listdir(directory_path):
# .png and Japanese # .png and Japanese
if '.png' in file_name and file_name.replace('.png', '') in VOCAB: if ".png" in file_name and file_name.replace(".png", "") in VOCAB:
file_path = os.path.join(directory_path, file_name) file_path = os.path.join(directory_path, file_name)
if os.path.isfile(file_path): if os.path.isfile(file_path):
# Check if the file is an image # Check if the file is an image
@ -206,7 +254,7 @@ def processImagesDir(directory_path, imageList):
width, height = getImageDimensions(file_path) width, height = getImageDimensions(file_path)
if width is not None and height is not None: if width is not None and height is not None:
placeholders = { placeholders = {
'.png': '', ".png": "",
} }
for target, replacement in placeholders.items(): for target, replacement in placeholders.items():
file_name = file_name.replace(target, replacement) file_name = file_name.replace(target, replacement)
@ -214,37 +262,49 @@ def processImagesDir(directory_path, imageList):
imageList[1].append([width, height]) imageList[1].append([width, height])
except Exception as e: except Exception as e:
print(f"Error processing {file_name}: {e}") print(f"Error processing {file_name}: {e}")
return imageList return imageList
def translateImages(imageList): def translateImages(imageList):
totalTokens = [0,0] totalTokens = [0, 0]
# Translate GPT # Translate GPT
response = translateGPT(imageList[0], 'Keep the Translation as brief as possible', True) response = translateGPT(
imageList[0], "Keep the Translation as brief as possible", True
)
translatedList = response[0] translatedList = response[0]
totalTokens[0] += response[1][0] totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1] totalTokens[1] += response[1][1]
return [translatedList, totalTokens, None] return [translatedList, totalTokens, None]
# Save some money and enter the character before translation # Save some money and enter the character before translation
def getSpeaker(speaker): def getSpeaker(speaker):
match speaker: match speaker:
case 'ファイン': case "ファイン":
return ['Fine', [0,0]] return ["Fine", [0, 0]]
case '': case "":
return ['', [0,0]] return ["", [0, 0]]
case _: case _:
# Store Speaker # Store Speaker
if speaker not in str(NAMESLIST): if speaker not in str(NAMESLIST):
response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False) response = translateGPT(
speaker,
"Reply with the " + LANGUAGE + " translation of the NPC name.",
False,
)
response[0] = response[0].title() response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s") response[0] = response[0].replace("'S", "'s")
# Retry if name doesn't translate for some reason # Retry if name doesn't translate for some reason
if re.search(r'([a-zA-Z？?])', response[0]) == None: if re.search(r"([a-zA-Z？?])", response[0]) == None:
response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False) response = translateGPT(
speaker,
"Reply with the " + LANGUAGE + " translation of the NPC name.",
False,
)
response[0] = response[0].title() response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s") response[0] = response[0].replace("'S", "'s")
@ -255,50 +315,56 @@ def getSpeaker(speaker):
else: else:
for i in range(len(NAMESLIST)): for i in range(len(NAMESLIST)):
if speaker == NAMESLIST[i][0]: if speaker == NAMESLIST[i][0]:
return [NAMESLIST[i][1],[0,0]] return [NAMESLIST[i][1], [0, 0]]
return [speaker,[0,0]] return [speaker, [0, 0]]
def subVars(jaString): def subVars(jaString):
jaString = jaString.replace('\u3000', ' ') jaString = jaString.replace("\u3000", " ")
# Formatting # Formatting
count = 0 count = 0
codeList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString) codeList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
codeList = set(codeList) codeList = set(codeList)
if len(codeList) != 0: if len(codeList) != 0:
for var in codeList: for var in codeList:
jaString = jaString.replace(var, '[FCode_' + str(count) + ']') jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1 count += 1
# Put all lists in list and return # Put all lists in list and return
return [jaString, codeList] return [jaString, codeList]
def resubVars(translatedText, codeList): def resubVars(translatedText, codeList):
# Fix Spacing and ChatGPT Nonsense # Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r'\[\s?.+?\s?\]', translatedText) matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0: if len(matchList) > 0:
for match in matchList: for match in matchList:
text = match.strip() text = match.strip()
translatedText = translatedText.replace(match, text) translatedText = translatedText.replace(match, text)
# Formatting # Formatting
count = 0 count = 0
if len(codeList) != 0: if len(codeList) != 0:
for var in codeList: for var in codeList:
translatedText = translatedText.replace('[FCode_' + str(count) + ']', var) translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1 count += 1
return translatedText return translatedText
def batchList(input_list, batch_size): def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0: if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer") raise ValueError("batch_size must be a positive integer")
return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)] return [
input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
]
def createContext(fullPromptFlag, subbedT, format): def createContext(fullPromptFlag, subbedT, format):
characters = 'Game Characters:\n\ characters = "Game Characters:\n\
ロラン (Roland) - Male\n\ ロラン (Roland) - Male\n\
リュカ (Ryuka) - Male\n\ リュカ (Ryuka) - Male\n\
レックス (Rex) - Male\n\ レックス (Rex) - Male\n\
@ -341,10 +407,12 @@ def createContext(fullPromptFlag, subbedT, format):
ブライ (Buraimu) - Male\n\ ブライ (Buraimu) - Male\n\
ハッサン (Hassan) - Male\n\ ハッサン (Hassan) - Male\n\
アロマ (Aroma) - Female\n\ アロマ (Aroma) - Female\n\
' "
system = PROMPT + VOCAB if fullPromptFlag else \ system = (
f"\ PROMPT + VOCAB
if fullPromptFlag
else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\ - You translate everything, including erotic and explicit content.\n\
@ -356,12 +424,14 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\ {VOCAB}\n\
" "
if format == 'json': )
user = f'```json\n{subbedT}\n```' if format == "json":
user = f"```json\n{subbedT}\n```"
else: else:
user = subbedT user = subbedT
return characters, system, user return characters, system, user
def translateText(characters, system, user, history, penalty, format): def translateText(characters, system, user, history, penalty, format):
# Prompt # Prompt
msg = [{"role": "system", "content": system + characters}] msg = [{"role": "system", "content": system + characters}]
@ -376,13 +446,13 @@ def translateText(characters, system, user, history, penalty, format):
msg.append({"role": "system", "content": history}) msg.append({"role": "system", "content": history})
# Response Format # Response Format
if format == 'json': if format == "json":
responseFormat = { "type": "json_object" } responseFormat = {"type": "json_object"}
else: else:
responseFormat = { "type": "text" } responseFormat = {"type": "text"}
# Content to TL # Content to TL
msg.append({"role": "user", "content": f'{user}'}) msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create( response = openai.chat.completions.create(
temperature=0, temperature=0,
frequency_penalty=penalty, frequency_penalty=penalty,
@ -392,18 +462,19 @@ def translateText(characters, system, user, history, penalty, format):
) )
return response return response
def cleanTranslatedText(translatedText, varResponse): def cleanTranslatedText(translatedText, varResponse):
placeholders = { placeholders = {
f'{LANGUAGE} Translation: ': '', f"{LANGUAGE} Translation: ": "",
'Translation: ': '', "Translation: ": "",
'っ': '', "っ": "",
'〜': '~', "〜": "~",
'ッ': '', "ッ": "",
'。': '.', "。": ".",
'「': '\\"', "「": '\\"',
'」': '\\"', "」": '\\"',
'- ': '-', "- ": "-",
'Placeholder Text': '', "Placeholder Text": "",
# Add more replacements as needed # Add more replacements as needed
} }
for target, replacement in placeholders.items(): for target, replacement in placeholders.items():
@ -414,11 +485,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1]) translatedText = resubVars(translatedText, varResponse[1])
return translatedText return translatedText
def elongateCharacters(text): def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters # Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character # Using a positive lookbehind assertion to capture the preceding character
pattern = r'(?<=(.))ー+' pattern = r"(?<=(.))ー+"
# Define a replacement function that elongates the captured character # Define a replacement function that elongates the captured character
def repl(match): def repl(match):
char = match.group(1) # The character before the ー sequence char = match.group(1) # The character before the ー sequence
@ -428,6 +500,7 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text # Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text) return re.sub(pattern, repl, text)
def extractTranslation(translatedTextList, is_list): def extractTranslation(translatedTextList, is_list):
try: try:
line_dict = json.loads(translatedTextList) line_dict = json.loads(translatedTextList)
@ -439,15 +512,15 @@ def extractTranslation(translatedTextList, is_list):
return string_list[0] return string_list[0]
except Exception as e: except Exception as e:
print(f'extractTranslation Error: {e}') print(f"extractTranslation Error: {e}")
return None return None
def countTokens(characters, system, user, history): def countTokens(characters, system, user, history):
inputTotalTokens = 0 inputTotalTokens = 0
outputTotalTokens = 0 outputTotalTokens = 0
enc = tiktoken.encoding_for_model('gpt-4') enc = tiktoken.encoding_for_model("gpt-4")
# Input # Input
if isinstance(history, list): if isinstance(history, list):
for line in history: for line in history:
@ -459,26 +532,28 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user)) inputTotalTokens += len(enc.encode(user))
# Output # Output
outputTotalTokens += round(len(enc.encode(user))*3) outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens] return [inputTotalTokens, outputTotalTokens]
def combineList(tlist, text): def combineList(tlist, text):
if isinstance(text, list): if isinstance(text, list):
return [t for sublist in tlist for t in sublist] return [t for sublist in tlist for t in sublist]
return tlist[0] return tlist[0]
@retry(exceptions=Exception, tries=5, delay=5) @retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag): def translateGPT(text, history, fullPromptFlag):
global PBAR global PBAR
mismatch = False mismatch = False
totalTokens = [0, 0] totalTokens = [0, 0]
if isinstance(text, list): if isinstance(text, list):
format = 'json' format = "json"
tList = batchList(text, BATCHSIZE) tList = batchList(text, BATCHSIZE)
else: else:
format = 'text' format = "text"
tList = [text] tList = [text]
for index, tItem in enumerate(tList): for index, tItem in enumerate(tList):
@ -493,7 +568,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0] subbedT = varResponse[0]
# Things to Check before starting translation # Things to Check before starting translation
if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT): if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
if PBAR is not None: if PBAR is not None:
PBAR.update(len(tItem)) PBAR.update(len(tItem))
continue continue
@ -518,9 +593,13 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse) translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list): if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True) extractedTranslations = extractTranslation(translatedText, True)
if extractedTranslations == None or len(tItem) != len(extractedTranslations): if extractedTranslations == None or len(tItem) != len(
extractedTranslations
):
# Mismatch. Try Again # Mismatch. Try Again
response = translateText(characters, system, user, history, 0.05, format) response = translateText(
characters, system, user, history, 0.05, format
)
translatedText = response.choices[0].message.content translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens totalTokens[1] += response.usage.completion_tokens
@ -529,13 +608,17 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse) translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list): if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True) extractedTranslations = extractTranslation(translatedText, True)
if extractedTranslations == None or len(tItem) != len(extractedTranslations): if extractedTranslations == None or len(tItem) != len(
mismatch = True # Just here for breakpoint extractedTranslations
):
mismatch = True # Just here for breakpoint
# Set if no mismatch # Set if no mismatch
if mismatch == False: if mismatch == False:
tList[index] = extractedTranslations tList[index] = extractedTranslations
history = extractedTranslations[-10:] # Update history if we have a list history = extractedTranslations[
-10:
] # Update history if we have a list
else: else:
history = text[-10:] history = text[-10:]
mismatch = False mismatch = False
@ -546,7 +629,7 @@ def translateGPT(text, history, fullPromptFlag):
PBAR.update(len(tItem)) PBAR.update(len(tItem))
else: else:
# Ensure we're passing a single string to extractTranslation # Ensure we're passing a single string to extractTranslation
tList[index] = translatedText tList[index] = translatedText
finalList = combineList(tList, text) finalList = combineList(tList, text)
return [finalList, totalTokens] return [finalList, totalTokens]

394
modules/irissoft.py
View file

@ -15,49 +15,50 @@ from tqdm import tqdm
# Open AI # Open AI
load_dotenv() load_dotenv()
if os.getenv('api').replace(' ', '') != '': if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv('api') openai.base_url = os.getenv("api")
openai.organization = os.getenv('org') openai.organization = os.getenv("org")
openai.api_key = os.getenv('key') openai.api_key = os.getenv("key")
#Globals # Globals
MODEL = os.getenv('model') MODEL = os.getenv("model")
TIMEOUT = int(os.getenv('timeout')) TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv('language').capitalize() LANGUAGE = os.getenv("language").capitalize()
PROMPT = Path('prompt.txt').read_text(encoding='utf-8') PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path('vocab.txt').read_text(encoding='utf-8') VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv('threads')) THREADS = int(os.getenv("threads"))
LOCK = threading.Lock() LOCK = threading.Lock()
WIDTH = int(os.getenv('width')) WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv('listWidth')) LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70 NOTEWIDTH = 70
MAXHISTORY = 10 MAXHISTORY = 10
ESTIMATE = '' ESTIMATE = ""
TOKENS = [0, 0] TOKENS = [0, 0]
NAMESLIST = [] NAMESLIST = []
NAMES = False # Output a list of all the character names found NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = True # Overwrites textwrap FIXTEXTWRAP = True # Overwrites textwrap
IGNORETLTEXT = False # Ignores all translated text. IGNORETLTEXT = False # Ignores all translated text.
MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
#tqdm Globals # tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0 POSITION = 0
LEAVE = False LEAVE = False
# Pricing - Depends on the model https://openai.com/pricing # Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request # Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size. # If you are getting a MISMATCH LENGTH error, lower the batch size.
if 'gpt-3.5' in MODEL: if "gpt-3.5" in MODEL:
INPUTAPICOST = .002 INPUTAPICOST = 0.002
OUTPUTAPICOST = .002 OUTPUTAPICOST = 0.002
BATCHSIZE = 10 BATCHSIZE = 10
elif 'gpt-4' in MODEL: elif "gpt-4" in MODEL:
INPUTAPICOST = .005 INPUTAPICOST = 0.005
OUTPUTAPICOST = .015 OUTPUTAPICOST = 0.015
BATCHSIZE = 40 BATCHSIZE = 40
def handleIris(filename, estimate): def handleIris(filename, estimate):
global ESTIMATE global ESTIMATE
ESTIMATE = estimate ESTIMATE = estimate
@ -74,17 +75,21 @@ def handleIris(filename, estimate):
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
# Print Total # Print Total
totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL') totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps # Print any errors on maps
if len(MISMATCH) > 0: if len(MISMATCH) > 0:
return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET return (
totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
)
else: else:
return totalString return totalString
else: else:
try: try:
with open('translated/' + filename, 'w', encoding='cp932', errors='ignore') as outFile: with open(
"translated/" + filename, "w", encoding="cp932", errors="ignore"
) as outFile:
start = time.time() start = time.time()
translatedData = openFiles(filename) translatedData = openFiles(filename)
@ -97,23 +102,36 @@ def handleIris(filename, estimate):
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
except Exception: except Exception:
traceback.print_exc() traceback.print_exc()
return 'Fail' return "Fail"
return getResultString(["", TOKENS, None], end - start, "TOTAL")
return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def getResultString(translatedData, translationTime, filename): def getResultString(translatedData, translationTime, filename):
# File Print String # File Print String
totalTokenstring =\ totalTokenstring = (
Fore.YELLOW +\ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
'[Input: ' + str(translatedData[1][0]) + ']'\ "[Output: "
'[Output: ' + str(translatedData[1][1]) + ']'\ + str(translatedData[1][1])
'[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\ + "]" "[Cost: ${:,.4f}".format(
(translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']' (translatedData[1][0] * 0.001 * INPUTAPICOST)
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
)
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None: if translatedData[2] == None:
# Success # Success
return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET return (
filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.GREEN
+ " \u2713 "
+ Fore.RESET
)
else: else:
# Fail # Fail
@ -122,31 +140,41 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
errorString = str(e) + Fore.RED errorString = str(e) + Fore.RED
return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\ return (
errorString + Fore.RESET filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.RED
+ " \u2717 "
+ errorString
+ Fore.RESET
)
def openFiles(filename): def openFiles(filename):
with open('files/' + filename, 'r', encoding='shift_jis') as readFile: with open("files/" + filename, "r", encoding="shift_jis") as readFile:
translatedData = parseIris(readFile, filename) translatedData = parseIris(readFile, filename)
# Delete lines marked for deletion # Delete lines marked for deletion
finalData = [] finalData = []
for line in translatedData[0]: for line in translatedData[0]:
if line != '\\d\n': if line != "\\d\n":
finalData.append(line) finalData.append(line)
translatedData[0] = finalData translatedData[0] = finalData
return translatedData return translatedData
def parseIris(readFile, filename): def parseIris(readFile, filename):
totalTokens = [0,0] totalTokens = [0, 0]
# Read File into data # Read File into data
data = readFile.readlines() data = readFile.readlines()
# Create Progress Bar # Create Progress Bar
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
pbar.desc=filename pbar.desc = filename
try: try:
result = translateIris(data, pbar, filename, []) result = translateIris(data, pbar, filename, [])
@ -157,83 +185,87 @@ def parseIris(readFile, filename):
return [data, totalTokens, e] return [data, totalTokens, e]
return [data, totalTokens, None] return [data, totalTokens, None]
def translateIris(data, pbar, filename, translatedList): def translateIris(data, pbar, filename, translatedList):
stringList = [] stringList = []
currentGroup = [] currentGroup = []
tokens = [0,0] tokens = [0, 0]
speaker = '' speaker = ""
voice = False voice = False
global LOCK, ESTIMATE global LOCK, ESTIMATE
i = 0 i = 0
while i < len(data): while i < len(data):
voice = False voice = False
speaker = '' speaker = ""
if '#MSGVOICE' in data[i]: if "#MSGVOICE" in data[i]:
i += 1 i += 1
voice = True voice = True
voiceVar = data[i] voiceVar = data[i]
if '#MSG,' in data[i] or '#MSG\n' in data[i] or voice == True: if "#MSG," in data[i] or "#MSG\n" in data[i] or voice == True:
i += 1 i += 1
# Speaker # Speaker
if re.search(r'^　?([^#\/."、。*!！（）\(\)\[\]　\n]+)\n', data[i]) and len(data[i]) < 30: if (
match = re.search(r'(.*)', data[i]) re.search(r'^　?([^#\/."、。*!！（）\(\)\[\]　\n]+)\n', data[i])
and len(data[i]) < 30
):
match = re.search(r"(.*)", data[i])
if match != None: if match != None:
speaker = match.group(1) speaker = match.group(1)
if speaker[0] == '\u3000': if speaker[0] == "\u3000":
speaker = speaker[1:] speaker = speaker[1:]
response = getSpeaker(speaker, pbar, filename) response = getSpeaker(speaker, pbar, filename)
speaker = response[0] speaker = response[0]
tokens[0] += response[1][0] tokens[0] += response[1][0]
tokens[1] += response[1][1] tokens[1] += response[1][1]
if translatedList != []: if translatedList != []:
speaker = speaker.replace(' ', '\u3000') speaker = speaker.replace(" ", "\u3000")
data[i] = f'\u3000{speaker}\n' data[i] = f"\u3000{speaker}\n"
else: else:
speaker = '' speaker = ""
i += 1 i += 1
# Lines # Lines
match = re.search(r'(.*)', data[i]) match = re.search(r"(.*)", data[i])
if match != None and match.group(1) != '': if match != None and match.group(1) != "":
# Pass 1 # Pass 1
if translatedList == []: if translatedList == []:
# Grab Consecutive Strings # Grab Consecutive Strings
jaString = data[i] jaString = data[i]
if data[i] != '\n': if data[i] != "\n":
if data[i][0] == '\u3000': if data[i][0] == "\u3000":
jaString = data[i][1:] jaString = data[i][1:]
currentGroup.append(jaString) currentGroup.append(jaString)
i += 1 i += 1
while data[i] != '\n': while data[i] != "\n":
jaString = data[i] jaString = data[i]
if data[i] != '\n': if data[i] != "\n":
jaString = data[i][1:] jaString = data[i][1:]
currentGroup.append(jaString) currentGroup.append(jaString)
i += 1 i += 1
# Join up 401 groups for better translation. # Join up 401 groups for better translation.
if len(currentGroup) > 0: if len(currentGroup) > 0:
jaString = ''.join(currentGroup) jaString = "".join(currentGroup)
currentGroup = [] currentGroup = []
# Remove any textwrap # Remove any textwrap
jaString = jaString.replace('\n', ' ') jaString = jaString.replace("\n", " ")
# Temporarily convert spaces (For Textwrap Later) # Temporarily convert spaces (For Textwrap Later)
jaString = jaString.replace('\u3000', ' ') jaString = jaString.replace("\u3000", " ")
# Add Speaker (If there is one) # Add Speaker (If there is one)
if speaker != '': if speaker != "":
jaString = f'{speaker}: {jaString}' jaString = f"{speaker}: {jaString}"
# Add String # Add String
stringList.append(jaString.strip()) stringList.append(jaString.strip())
# Pass 2 # Pass 2
else: else:
# Insert Strings # Insert Strings
while data[i] != '\n': while data[i] != "\n":
data.pop(i) data.pop(i)
# Get Text # Get Text
@ -244,21 +276,21 @@ def translateIris(data, pbar, filename, translatedList):
translatedList = None translatedList = None
# Remove added speaker # Remove added speaker
translatedText = re.sub(r'^.+?:\s', '', translatedText) translatedText = re.sub(r"^.+?:\s", "", translatedText)
# Textwrap # Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH) translatedText = textwrap.fill(translatedText, width=WIDTH)
translatedText = translatedText.replace('\n', '\n\u3000') translatedText = translatedText.replace("\n", "\n\u3000")
# Replace Whitespace and Commas # Replace Whitespace and Commas
translatedText = translatedText.replace(', ', '、') translatedText = translatedText.replace(", ", "、")
translatedText = translatedText.replace(',\u3000', '、') translatedText = translatedText.replace(",\u3000", "、")
translatedText = translatedText.replace(',', '、') translatedText = translatedText.replace(",", "、")
translatedText = translatedText.replace(' ', '\u3000') translatedText = translatedText.replace(" ", "\u3000")
# Set Data # Set Data
# Game crashes on more than 3 lines. Will need to create a new MSG for long translations # Game crashes on more than 3 lines. Will need to create a new MSG for long translations
if translatedText.count('\n') > 2: if translatedText.count("\n") > 2:
# Split List # Split List
translatedTextList = splitNewlines(translatedText) translatedTextList = splitNewlines(translatedText)
@ -267,34 +299,34 @@ def translateIris(data, pbar, filename, translatedList):
for text in translatedTextList: for text in translatedTextList:
if count != 0: if count != 0:
if voice == True: if voice == True:
#MSG for each item in the list # MSG for each item in the list
data.insert(i, '#MSGVOICE,\n') data.insert(i, "#MSGVOICE,\n")
i += 1 i += 1
data.insert(i, f'{voiceVar}') data.insert(i, f"{voiceVar}")
i += 1 i += 1
else: else:
data.insert(i, '#MSG,\n') data.insert(i, "#MSG,\n")
i += 1 i += 1
if speaker: if speaker:
data[i] = f'\u3000{speaker}\n' data[i] = f"\u3000{speaker}\n"
i += 1 i += 1
if text[0] == '\u3000': if text[0] == "\u3000":
data.insert(i, f'{text}\n') data.insert(i, f"{text}\n")
else: else:
data.insert(i, f'\u3000{text}\n') data.insert(i, f"\u3000{text}\n")
i += 1 i += 1
count += 1 count += 1
if data[i] != '\n': if data[i] != "\n":
data.insert(i, '\n') data.insert(i, "\n")
data[i] = f'\n{data[i]}' data[i] = f"\n{data[i]}"
else: else:
data.insert(i, f'\u3000{translatedText}\n') data.insert(i, f"\u3000{translatedText}\n")
i += 1 i += 1
if data[i] != '\n': if data[i] != "\n":
data[i] = f'\n{data[i]}' data[i] = f"\n{data[i]}"
elif '#SELECT' in data[i] and translatedList == []: elif "#SELECT" in data[i] and translatedList == []:
Iris = r'(.+?) +\d$' Iris = r"(.+?) +\d$"
i += 1 i += 1
match = re.search(Iris, data[i]) match = re.search(Iris, data[i])
if match: if match:
@ -302,14 +334,20 @@ def translateIris(data, pbar, filename, translatedList):
choiceList.append(match.group(1)) choiceList.append(match.group(1))
i += 1 i += 1
match = re.search(Iris, data[i]) match = re.search(Iris, data[i])
while(match): while match:
choiceList.append(match.group(1)) choiceList.append(match.group(1))
i += 1 i += 1
match = re.search(Iris, data[i]) match = re.search(Iris, data[i])
# Translate # Translate
question = stringList[len(stringList) - 1] question = stringList[len(stringList) - 1]
response = translateGPT(choiceList, f'Previous text for context: {question}\n\nThis will be a dialogue option', True, pbar, filename) response = translateGPT(
choiceList,
f"Previous text for context: {question}\n\nThis will be a dialogue option",
True,
pbar,
filename,
)
tokens[0] += response[1][0] tokens[0] += response[1][0]
tokens[1] += response[1][1] tokens[1] += response[1][1]
choiceListTL = response[0] choiceListTL = response[0]
@ -318,10 +356,10 @@ def translateIris(data, pbar, filename, translatedList):
i = i - len(choiceListTL) i = i - len(choiceListTL)
for j in range(len(choiceListTL)): for j in range(len(choiceListTL)):
# Replace Whitespace and Commas # Replace Whitespace and Commas
choiceListTL[j] = choiceListTL[j].replace(', ', '、') choiceListTL[j] = choiceListTL[j].replace(", ", "、")
choiceListTL[j] = choiceListTL[j].replace(',\u3000', '、') choiceListTL[j] = choiceListTL[j].replace(",\u3000", "、")
choiceListTL[j] = choiceListTL[j].replace(',', '、') choiceListTL[j] = choiceListTL[j].replace(",", "、")
choiceListTL[j] = choiceListTL[j].replace(' ', '\u3000') choiceListTL[j] = choiceListTL[j].replace(" ", "\u3000")
data[i] = data[i].replace(choiceList[j], choiceListTL[j]) data[i] = data[i].replace(choiceList[j], choiceListTL[j])
i += 1 i += 1
@ -336,9 +374,9 @@ def translateIris(data, pbar, filename, translatedList):
# Set Progress # Set Progress
pbar.total = len(stringList) pbar.total = len(stringList)
pbar.refresh() pbar.refresh()
# Translate # Translate
response = translateGPT(stringList, '', True, pbar, filename) response = translateGPT(stringList, "", True, pbar, filename)
tokens[0] += response[1][0] tokens[0] += response[1][0]
tokens[1] += response[1][1] tokens[1] += response[1][1]
translatedList = response[0] translatedList = response[0]
@ -354,20 +392,21 @@ def translateIris(data, pbar, filename, translatedList):
MISMATCH.append(filename) MISMATCH.append(filename)
return tokens return tokens
def splitNewlines(text): def splitNewlines(text):
parts = [] parts = []
newline_count = 0 # Counts the number of newline characters encountered newline_count = 0 # Counts the number of newline characters encountered
start_index = 0 # Start index of the current string part start_index = 0 # Start index of the current string part
for i, char in enumerate(text): for i, char in enumerate(text):
if char == '\n': if char == "\n":
newline_count += 1 newline_count += 1
if newline_count == 3: if newline_count == 3:
# Append the string part from start_index to current index (inclusive) # Append the string part from start_index to current index (inclusive)
parts.append(text[start_index:i+1]) parts.append(text[start_index : i + 1])
# Reset newline count and update start_index for the next string part # Reset newline count and update start_index for the next string part
newline_count = 0 newline_count = 0
start_index = i+1 start_index = i + 1
# Edge case: if the text does not end with a newline, we still need to append the last part # Edge case: if the text does not end with a newline, we still need to append the last part
if start_index < len(text): if start_index < len(text):
@ -375,94 +414,103 @@ def splitNewlines(text):
return parts return parts
# Save some money and enter the character before translation # Save some money and enter the character before translation
def getSpeaker(speaker, pbar, filename): def getSpeaker(speaker, pbar, filename):
match speaker: match speaker:
case 'ファイン': case "ファイン":
return ['Fine', [0,0]] return ["Fine", [0, 0]]
case '': case "":
return ['', [0,0]] return ["", [0, 0]]
case _: case _:
# Store Speaker # Store Speaker
if speaker not in str(NAMESLIST): if speaker not in str(NAMESLIST):
response = translateGPT(speaker, 'Reply with only the '+ LANGUAGE +' translation of the NPC name.', False, pbar, filename) response = translateGPT(
speaker,
"Reply with only the " + LANGUAGE + " translation of the NPC name.",
False,
pbar,
filename,
)
response[0] = response[0].replace("'S", "'s") response[0] = response[0].replace("'S", "'s")
speakerList = [speaker, response[0]] speakerList = [speaker, response[0]]
NAMESLIST.append(speakerList) NAMESLIST.append(speakerList)
return response return response
# Find Speaker # Find Speaker
else: else:
for i in range(len(NAMESLIST)): for i in range(len(NAMESLIST)):
if speaker == NAMESLIST[i][0]: if speaker == NAMESLIST[i][0]:
return [NAMESLIST[i][1],[0,0]] return [NAMESLIST[i][1], [0, 0]]
return [speaker,[0,0]] return [speaker, [0, 0]]
def subVars(jaString): def subVars(jaString):
jaString = jaString.replace('\u3000', ' ') jaString = jaString.replace("\u3000", " ")
# Nested # Nested
count = 0 count = 0
nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString) nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList) nestedList = set(nestedList)
if len(nestedList) != 0: if len(nestedList) != 0:
for icon in nestedList: for icon in nestedList:
jaString = jaString.replace(icon, '[Nested_' + str(count) + ']') jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString) iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList) iconList = set(iconList)
if len(iconList) != 0: if len(iconList) != 0:
for icon in iconList: for icon in iconList:
jaString = jaString.replace(icon, '[Ascii_' + str(count) + ']') jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString) colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList) colorList = set(colorList)
if len(colorList) != 0: if len(colorList) != 0:
for color in colorList: for color in colorList:
jaString = jaString.replace(color, '[Color_' + str(count) + ']') jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1 count += 1
# Names # Names
count = 0 count = 0
nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString) nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList) nameList = set(nameList)
if len(nameList) != 0: if len(nameList) != 0:
for name in nameList: for name in nameList:
jaString = jaString.replace(name, '[Noun_' + str(count) + ']') jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1 count += 1
# Variables # Variables
count = 0 count = 0
varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString) varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList) varList = set(varList)
if len(varList) != 0: if len(varList) != 0:
for var in varList: for var in varList:
jaString = jaString.replace(var, '[Var_' + str(count) + ']') jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
formatList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString) formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList) formatList = set(formatList)
if len(formatList) != 0: if len(formatList) != 0:
for var in formatList: for var in formatList:
jaString = jaString.replace(var, '[FCode_' + str(count) + ']') jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1 count += 1
# Put all lists in list and return # Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList] allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList] return [jaString, allList]
def resubVars(translatedText, allList): def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense # Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r'\[\s?.+?\s?\]', translatedText) matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0: if len(matchList) > 0:
for match in matchList: for match in matchList:
text = match.strip() text = match.strip()
@ -472,54 +520,58 @@ def resubVars(translatedText, allList):
count = 0 count = 0
if len(allList[0]) != 0: if len(allList[0]) != 0:
for var in allList[0]: for var in allList[0]:
translatedText = translatedText.replace('[Nested_' + str(count) + ']', var) translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
if len(allList[1]) != 0: if len(allList[1]) != 0:
for var in allList[1]: for var in allList[1]:
translatedText = translatedText.replace('[Ascii_' + str(count) + ']', var) translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
if len(allList[2]) != 0: if len(allList[2]) != 0:
for var in allList[2]: for var in allList[2]:
translatedText = translatedText.replace('[Color_' + str(count) + ']', var) translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1 count += 1
# Names # Names
count = 0 count = 0
if len(allList[3]) != 0: if len(allList[3]) != 0:
for var in allList[3]: for var in allList[3]:
translatedText = translatedText.replace('[Noun_' + str(count) + ']', var) translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1 count += 1
# Vars # Vars
count = 0 count = 0
if len(allList[4]) != 0: if len(allList[4]) != 0:
for var in allList[4]: for var in allList[4]:
translatedText = translatedText.replace('[Var_' + str(count) + ']', var) translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
if len(allList[5]) != 0: if len(allList[5]) != 0:
for var in allList[5]: for var in allList[5]:
translatedText = translatedText.replace('[FCode_' + str(count) + ']', var) translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1 count += 1
return translatedText return translatedText
def batchList(input_list, batch_size): def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0: if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer") raise ValueError("batch_size must be a positive integer")
return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)] return [
input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
]
def createContext(fullPromptFlag, subbedT): def createContext(fullPromptFlag, subbedT):
characters = 'Game Characters:\n\ characters = "Game Characters:\n\
フィリア (Philia) - Female\n\ フィリア (Philia) - Female\n\
アルネット (Annett) - Female\n\ アルネット (Annett) - Female\n\
ラピュセナ (Rapusena) - Female\n\ ラピュセナ (Rapusena) - Female\n\
@ -529,10 +581,12 @@ def createContext(fullPromptFlag, subbedT):
カルナ (Karna) - Female\n\ カルナ (Karna) - Female\n\
ラフィング＝スピア (Laughing Spear) - Female\n\ ラフィング＝スピア (Laughing Spear) - Female\n\
ノーラ (Nora) - Female\n\ ノーラ (Nora) - Female\n\
' "
system = PROMPT + VOCAB if fullPromptFlag else \ system = (
f"\ PROMPT + VOCAB
if fullPromptFlag
else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\ - You translate everything, including erotic and explicit content.\n\
@ -544,9 +598,11 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\ {VOCAB}\n\
" "
user = f'{subbedT}' )
user = f"{subbedT}"
return characters, system, user return characters, system, user
def translateText(characters, system, user, history): def translateText(characters, system, user, history):
# Prompt # Prompt
msg = [{"role": "system", "content": system + characters}] msg = [{"role": "system", "content": system + characters}]
@ -559,9 +615,9 @@ def translateText(characters, system, user, history):
msg.extend([{"role": "system", "content": h} for h in history]) msg.extend([{"role": "system", "content": h} for h in history])
else: else:
msg.append({"role": "system", "content": history}) msg.append({"role": "system", "content": history})
# Content to TL # Content to TL
msg.append({"role": "user", "content": f'{user}'}) msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create( response = openai.chat.completions.create(
temperature=0.1, temperature=0.1,
frequency_penalty=0.1, frequency_penalty=0.1,
@ -570,15 +626,16 @@ def translateText(characters, system, user, history):
) )
return response return response
def cleanTranslatedText(translatedText, varResponse): def cleanTranslatedText(translatedText, varResponse):
placeholders = { placeholders = {
f'{LANGUAGE} Translation: ': '', f"{LANGUAGE} Translation: ": "",
'Translation: ': '', "Translation: ": "",
'っ': '', "っ": "",
'〜': '~', "〜": "~",
'ッ': '', "ッ": "",
'。': '.', "。": ".",
'Placeholder Text': '' "Placeholder Text": "",
# Add more replacements as needed # Add more replacements as needed
} }
for target, replacement in placeholders.items(): for target, replacement in placeholders.items():
@ -589,11 +646,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1]) translatedText = resubVars(translatedText, varResponse[1])
return translatedText return translatedText
def elongateCharacters(text): def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters # Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character # Using a positive lookbehind assertion to capture the preceding character
pattern = r'(?<=(.))ー+' pattern = r"(?<=(.))ー+"
# Define a replacement function that elongates the captured character # Define a replacement function that elongates the captured character
def repl(match): def repl(match):
char = match.group(1) # The character before the ー sequence char = match.group(1) # The character before the ー sequence
@ -603,8 +661,9 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text # Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text) return re.sub(pattern, repl, text)
def extractTranslation(translatedTextList, is_list): def extractTranslation(translatedTextList, is_list):
pattern = r'`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?' pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item. # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list: if is_list:
matchList = re.findall(pattern, translatedTextList) matchList = re.findall(pattern, translatedTextList)
@ -613,11 +672,12 @@ def extractTranslation(translatedTextList, is_list):
matchList = re.findall(pattern, translatedTextList) matchList = re.findall(pattern, translatedTextList)
return matchList[0][0] if matchList else translatedTextList return matchList[0][0] if matchList else translatedTextList
def countTokens(characters, system, user, history): def countTokens(characters, system, user, history):
inputTotalTokens = 0 inputTotalTokens = 0
outputTotalTokens = 0 outputTotalTokens = 0
enc = tiktoken.encoding_for_model('gpt-4') enc = tiktoken.encoding_for_model("gpt-4")
# Input # Input
if isinstance(history, list): if isinstance(history, list):
for line in history: for line in history:
@ -629,15 +689,17 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user)) inputTotalTokens += len(enc.encode(user))
# Output # Output
outputTotalTokens += round(len(enc.encode(user))*2) outputTotalTokens += round(len(enc.encode(user)) * 2)
return [inputTotalTokens, outputTotalTokens] return [inputTotalTokens, outputTotalTokens]
def combineList(tlist, text): def combineList(tlist, text):
if isinstance(text, list): if isinstance(text, list):
return [t for sublist in tlist for t in sublist] return [t for sublist in tlist for t in sublist]
return tlist[0] return tlist[0]
@retry(exceptions=Exception, tries=5, delay=5) @retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag, pbar, filename): def translateGPT(text, history, fullPromptFlag, pbar, filename):
mismatch = False mismatch = False
@ -650,8 +712,12 @@ def translateGPT(text, history, fullPromptFlag, pbar, filename):
for index, tItem in enumerate(tList): for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting # Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list): if isinstance(tItem, list):
payload = '\n'.join([f'`<Line{i}>{item}</Line{i}>`' for i, item in enumerate(tItem)]) payload = "\n".join(
payload = re.sub(r'(<Line\d+)(><)(\/Line\d+>)', r'\1>Placeholder Text<\3', payload) [f"`<Line{i}>{item}</Line{i}>`" for i, item in enumerate(tItem)]
)
payload = re.sub(
r"(<Line\d+)(><)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload
)
varResponse = subVars(payload) varResponse = subVars(payload)
subbedT = varResponse[0] subbedT = varResponse[0]
else: else:
@ -659,7 +725,7 @@ def translateGPT(text, history, fullPromptFlag, pbar, filename):
subbedT = varResponse[0] subbedT = varResponse[0]
# Things to Check before starting translation # Things to Check before starting translation
if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT): if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue continue
# Create Message # Create Message

252
modules/javascript.py
View file

@ -15,49 +15,50 @@ from tqdm import tqdm
# Open AI # Open AI
load_dotenv() load_dotenv()
if os.getenv('api').replace(' ', '') != '': if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv('api') openai.base_url = os.getenv("api")
openai.organization = os.getenv('org') openai.organization = os.getenv("org")
openai.api_key = os.getenv('key') openai.api_key = os.getenv("key")
#Globals # Globals
MODEL = os.getenv('model') MODEL = os.getenv("model")
TIMEOUT = int(os.getenv('timeout')) TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv('language').capitalize() LANGUAGE = os.getenv("language").capitalize()
PROMPT = Path('prompt.txt').read_text(encoding='utf-8') PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path('vocab.txt').read_text(encoding='utf-8') VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv('threads')) THREADS = int(os.getenv("threads"))
LOCK = threading.Lock() LOCK = threading.Lock()
WIDTH = int(os.getenv('width')) WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv('listWidth')) LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70 NOTEWIDTH = 70
MAXHISTORY = 10 MAXHISTORY = 10
ESTIMATE = '' ESTIMATE = ""
TOKENS = [0, 0] TOKENS = [0, 0]
NAMESLIST = [] NAMESLIST = []
NAMES = False # Output a list of all the character names found NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = True # Overwrites textwrap FIXTEXTWRAP = True # Overwrites textwrap
IGNORETLTEXT = False # Ignores all translated text. IGNORETLTEXT = False # Ignores all translated text.
MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
#tqdm Globals # tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0 POSITION = 0
LEAVE = False LEAVE = False
# Pricing - Depends on the model https://openai.com/pricing # Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request # Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size. # If you are getting a MISMATCH LENGTH error, lower the batch size.
if 'gpt-3.5' in MODEL: if "gpt-3.5" in MODEL:
INPUTAPICOST = .002 INPUTAPICOST = 0.002
OUTPUTAPICOST = .002 OUTPUTAPICOST = 0.002
BATCHSIZE = 10 BATCHSIZE = 10
elif 'gpt-4' in MODEL: elif "gpt-4" in MODEL:
INPUTAPICOST = .005 INPUTAPICOST = 0.005
OUTPUTAPICOST = .015 OUTPUTAPICOST = 0.015
BATCHSIZE = 40 BATCHSIZE = 40
def handleJavascript(filename, estimate): def handleJavascript(filename, estimate):
global ESTIMATE global ESTIMATE
ESTIMATE = estimate ESTIMATE = estimate
@ -74,17 +75,21 @@ def handleJavascript(filename, estimate):
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
# Print Total # Print Total
totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL') totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps # Print any errors on maps
if len(MISMATCH) > 0: if len(MISMATCH) > 0:
return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET return (
totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
)
else: else:
return totalString return totalString
else: else:
try: try:
with open('translated/' + filename, 'w', encoding='utf8', errors='ignore') as outFile: with open(
"translated/" + filename, "w", encoding="utf8", errors="ignore"
) as outFile:
start = time.time() start = time.time()
translatedData = openFiles(filename) translatedData = openFiles(filename)
@ -97,23 +102,36 @@ def handleJavascript(filename, estimate):
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
except Exception: except Exception:
traceback.print_exc() traceback.print_exc()
return 'Fail' return "Fail"
return getResultString(["", TOKENS, None], end - start, "TOTAL")
return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def getResultString(translatedData, translationTime, filename): def getResultString(translatedData, translationTime, filename):
# File Print String # File Print String
totalTokenstring =\ totalTokenstring = (
Fore.YELLOW +\ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
'[Input: ' + str(translatedData[1][0]) + ']'\ "[Output: "
'[Output: ' + str(translatedData[1][1]) + ']'\ + str(translatedData[1][1])
'[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\ + "]" "[Cost: ${:,.4f}".format(
(translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']' (translatedData[1][0] * 0.001 * INPUTAPICOST)
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
)
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None: if translatedData[2] == None:
# Success # Success
return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET return (
filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.GREEN
+ " \u2713 "
+ Fore.RESET
)
else: else:
# Fail # Fail
@ -122,21 +140,31 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
errorString = str(e) + Fore.RED errorString = str(e) + Fore.RED
return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\ return (
errorString + Fore.RESET filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.RED
+ " \u2717 "
+ errorString
+ Fore.RESET
)
def openFiles(filename): def openFiles(filename):
with open('files/' + filename, 'r', encoding='utf-8') as readFile: with open("files/" + filename, "r", encoding="utf-8") as readFile:
translatedData = parseJS(readFile, filename) translatedData = parseJS(readFile, filename)
return translatedData return translatedData
def parseJS(readFile, filename): def parseJS(readFile, filename):
totalTokens = [0,0] totalTokens = [0, 0]
data = readFile.readlines() data = readFile.readlines()
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
pbar.desc=filename pbar.desc = filename
try: try:
result = translateJS(data, pbar) result = translateJS(data, pbar)
@ -147,8 +175,9 @@ def parseJS(readFile, filename):
return [data, totalTokens, e] return [data, totalTokens, e]
return [data, totalTokens, None] return [data, totalTokens, None]
def translateJS(data, pbar): def translateJS(data, pbar):
tokens = [0,0] tokens = [0, 0]
i = 0 i = 0
# Regex & Plugin Name # Regex & Plugin Name
@ -165,10 +194,14 @@ def translateJS(data, pbar):
# Remove Wordwrap [Optional] # Remove Wordwrap [Optional]
for j in range(len(modifiedStringList)): for j in range(len(modifiedStringList)):
modifiedStringList[j] = modifiedStringList[j].replace(r'\\\\\\\\n', r' ') modifiedStringList[j] = modifiedStringList[j].replace(
r"\\\\\\\\n", r" "
)
# Translate # Translate
response = translateGPT(modifiedStringList, f'Reply with the {LANGUAGE} translation', True, pbar) response = translateGPT(
modifiedStringList, f"Reply with the {LANGUAGE} translation", True, pbar
)
translatedList = response[0] translatedList = response[0]
tokens[0] = response[1][0] tokens[0] = response[1][0]
tokens[0] = response[1][1] tokens[0] = response[1][1]
@ -181,82 +214,83 @@ def translateJS(data, pbar):
# Wordwrap [Optional] # Wordwrap [Optional]
translatedList[j] = textwrap.fill(translatedList[j], LISTWIDTH) translatedList[j] = textwrap.fill(translatedList[j], LISTWIDTH)
translatedList[j] = translatedList[j].replace('\n', r'\\\\\\\\n') translatedList[j] = translatedList[j].replace("\n", r"\\\\\\\\n")
# Set # Set
data[i] = data[i].replace(stringList[j], translatedList[j]) data[i] = data[i].replace(stringList[j], translatedList[j])
# Mismatch # Mismatch
else: else:
pbar.write('Mismatch Error') pbar.write("Mismatch Error")
i += 1 i += 1
return tokens return tokens
def subVars(jaString): def subVars(jaString):
jaString = jaString.replace('\u3000', ' ') jaString = jaString.replace("\u3000", " ")
# Nested # Nested
count = 0 count = 0
nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString) nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList) nestedList = set(nestedList)
if len(nestedList) != 0: if len(nestedList) != 0:
for icon in nestedList: for icon in nestedList:
jaString = jaString.replace(icon, '[Nested_' + str(count) + ']') jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString) iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList) iconList = set(iconList)
if len(iconList) != 0: if len(iconList) != 0:
for icon in iconList: for icon in iconList:
jaString = jaString.replace(icon, '[Ascii_' + str(count) + ']') jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString) colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList) colorList = set(colorList)
if len(colorList) != 0: if len(colorList) != 0:
for color in colorList: for color in colorList:
jaString = jaString.replace(color, '[Color_' + str(count) + ']') jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1 count += 1
# Names # Names
count = 0 count = 0
nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString) nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList) nameList = set(nameList)
if len(nameList) != 0: if len(nameList) != 0:
for name in nameList: for name in nameList:
jaString = jaString.replace(name, '[Noun_' + str(count) + ']') jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1 count += 1
# Variables # Variables
count = 0 count = 0
varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString) varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList) varList = set(varList)
if len(varList) != 0: if len(varList) != 0:
for var in varList: for var in varList:
jaString = jaString.replace(var, '[Var_' + str(count) + ']') jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
formatList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString) formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList) formatList = set(formatList)
if len(formatList) != 0: if len(formatList) != 0:
for var in formatList: for var in formatList:
jaString = jaString.replace(var, '[FCode_' + str(count) + ']') jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1 count += 1
# Put all lists in list and return # Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList] allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList] return [jaString, allList]
def resubVars(translatedText, allList): def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense # Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r'\[\s?.+?\s?\]', translatedText) matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0: if len(matchList) > 0:
for match in matchList: for match in matchList:
text = match.strip() text = match.strip()
@ -266,64 +300,70 @@ def resubVars(translatedText, allList):
count = 0 count = 0
if len(allList[0]) != 0: if len(allList[0]) != 0:
for var in allList[0]: for var in allList[0]:
translatedText = translatedText.replace('[Nested_' + str(count) + ']', var) translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
if len(allList[1]) != 0: if len(allList[1]) != 0:
for var in allList[1]: for var in allList[1]:
translatedText = translatedText.replace('[Ascii_' + str(count) + ']', var) translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
if len(allList[2]) != 0: if len(allList[2]) != 0:
for var in allList[2]: for var in allList[2]:
translatedText = translatedText.replace('[Color_' + str(count) + ']', var) translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1 count += 1
# Names # Names
count = 0 count = 0
if len(allList[3]) != 0: if len(allList[3]) != 0:
for var in allList[3]: for var in allList[3]:
translatedText = translatedText.replace('[Noun_' + str(count) + ']', var) translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1 count += 1
# Vars # Vars
count = 0 count = 0
if len(allList[4]) != 0: if len(allList[4]) != 0:
for var in allList[4]: for var in allList[4]:
translatedText = translatedText.replace('[Var_' + str(count) + ']', var) translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
if len(allList[5]) != 0: if len(allList[5]) != 0:
for var in allList[5]: for var in allList[5]:
translatedText = translatedText.replace('[FCode_' + str(count) + ']', var) translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1 count += 1
return translatedText return translatedText
def batchList(input_list, batch_size): def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0: if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer") raise ValueError("batch_size must be a positive integer")
return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)] return [
input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
]
def createContext(fullPromptFlag, subbedT): def createContext(fullPromptFlag, subbedT):
characters = 'Game Characters:\n\ characters = "Game Characters:\n\
皆月 (Minazuki)\n\ 皆月 (Minazuki)\n\
さやか (Sayaka)\n\ さやか (Sayaka)\n\
皆月 さやか (Minazuki Sayaka) - Female\n\ 皆月 さやか (Minazuki Sayaka) - Female\n\
広瀬 (Hirose)\n\ 広瀬 (Hirose)\n\
智恵 (Chie) - Female\n\ 智恵 (Chie) - Female\n\
広瀬 智恵 (Hirose Chie) - Female\n\ 広瀬 智恵 (Hirose Chie) - Female\n\
' "
system = PROMPT + VOCAB if fullPromptFlag else \ system = (
f"\ PROMPT + VOCAB
if fullPromptFlag
else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\ - You translate everything, including erotic and explicit content.\n\
@ -335,9 +375,11 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\ {VOCAB}\n\
" "
user = f'{subbedT}' )
user = f"{subbedT}"
return characters, system, user return characters, system, user
def translateText(characters, system, user, history): def translateText(characters, system, user, history):
# Prompt # Prompt
msg = [{"role": "system", "content": system + characters}] msg = [{"role": "system", "content": system + characters}]
@ -350,9 +392,9 @@ def translateText(characters, system, user, history):
msg.extend([{"role": "system", "content": h} for h in history]) msg.extend([{"role": "system", "content": h} for h in history])
else: else:
msg.append({"role": "system", "content": history}) msg.append({"role": "system", "content": history})
# Content to TL # Content to TL
msg.append({"role": "user", "content": f'{user}'}) msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create( response = openai.chat.completions.create(
temperature=0.1, temperature=0.1,
frequency_penalty=0.1, frequency_penalty=0.1,
@ -361,15 +403,16 @@ def translateText(characters, system, user, history):
) )
return response return response
def cleanTranslatedText(translatedText, varResponse): def cleanTranslatedText(translatedText, varResponse):
placeholders = { placeholders = {
f'{LANGUAGE} Translation: ': '', f"{LANGUAGE} Translation: ": "",
'Translation: ': '', "Translation: ": "",
'っ': '', "っ": "",
'〜': '~', "〜": "~",
'ッ': '', "ッ": "",
'。': '.', "。": ".",
'Placeholder Text': '' "Placeholder Text": "",
# Add more replacements as needed # Add more replacements as needed
} }
for target, replacement in placeholders.items(): for target, replacement in placeholders.items():
@ -380,11 +423,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1]) translatedText = resubVars(translatedText, varResponse[1])
return translatedText return translatedText
def elongateCharacters(text): def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters # Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character # Using a positive lookbehind assertion to capture the preceding character
pattern = r'(?<=(.))ー+' pattern = r"(?<=(.))ー+"
# Define a replacement function that elongates the captured character # Define a replacement function that elongates the captured character
def repl(match): def repl(match):
char = match.group(1) # The character before the ー sequence char = match.group(1) # The character before the ー sequence
@ -394,8 +438,9 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text # Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text) return re.sub(pattern, repl, text)
def extractTranslation(translatedTextList, is_list): def extractTranslation(translatedTextList, is_list):
pattern = r'`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?' pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item. # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list: if is_list:
matchList = re.findall(pattern, translatedTextList) matchList = re.findall(pattern, translatedTextList)
@ -404,11 +449,12 @@ def extractTranslation(translatedTextList, is_list):
matchList = re.findall(pattern, translatedTextList) matchList = re.findall(pattern, translatedTextList)
return matchList[0][0] if matchList else translatedTextList return matchList[0][0] if matchList else translatedTextList
def countTokens(characters, system, user, history): def countTokens(characters, system, user, history):
inputTotalTokens = 0 inputTotalTokens = 0
outputTotalTokens = 0 outputTotalTokens = 0
enc = tiktoken.encoding_for_model('gpt-4') enc = tiktoken.encoding_for_model("gpt-4")
# Input # Input
if isinstance(history, list): if isinstance(history, list):
for line in history: for line in history:
@ -420,15 +466,17 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user)) inputTotalTokens += len(enc.encode(user))
# Output # Output
outputTotalTokens += round(len(enc.encode(user))*3) outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens] return [inputTotalTokens, outputTotalTokens]
def combineList(tlist, text): def combineList(tlist, text):
if isinstance(text, list): if isinstance(text, list):
return [t for sublist in tlist for t in sublist] return [t for sublist in tlist for t in sublist]
return tlist[0] return tlist[0]
@retry(exceptions=Exception, tries=5, delay=5) @retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag, pbar): def translateGPT(text, history, fullPromptFlag, pbar):
mismatch = False mismatch = False
@ -441,8 +489,12 @@ def translateGPT(text, history, fullPromptFlag, pbar):
for index, tItem in enumerate(tList): for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting # Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list): if isinstance(tItem, list):
payload = '\n'.join([f'`<Line{i}>{item}</Line{i}>`' for i, item in enumerate(tItem)]) payload = "\n".join(
payload = re.sub(r'(<Line\d+)(><)(\/Line\d+>)', r'\1>Placeholder Text<\3', payload) [f"`<Line{i}>{item}</Line{i}>`" for i, item in enumerate(tItem)]
)
payload = re.sub(
r"(<Line\d+)(><)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload
)
varResponse = subVars(payload) varResponse = subVars(payload)
subbedT = varResponse[0] subbedT = varResponse[0]
else: else:
@ -450,7 +502,7 @@ def translateGPT(text, history, fullPromptFlag, pbar):
subbedT = varResponse[0] subbedT = varResponse[0]
# Things to Check before starting translation # Things to Check before starting translation
if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT): if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue continue
# Create Message # Create Message
@ -488,7 +540,7 @@ def translateGPT(text, history, fullPromptFlag, pbar):
if len(tItem) == len(extractedTranslations): if len(tItem) == len(extractedTranslations):
tList[index] = extractedTranslations tList[index] = extractedTranslations
else: else:
mismatch = True # Just here for breakpoint mismatch = True # Just here for breakpoint
# Create History # Create History
history = tList[index] # Update history if we have a list history = tList[index] # Update history if we have a list

316
modules/json.py
View file

@ -16,49 +16,50 @@ from tqdm import tqdm
# Open AI # Open AI
load_dotenv() load_dotenv()
if os.getenv('api').replace(' ', '') != '': if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv('api') openai.base_url = os.getenv("api")
openai.organization = os.getenv('org') openai.organization = os.getenv("org")
openai.api_key = os.getenv('key') openai.api_key = os.getenv("key")
#Globals # Globals
MODEL = os.getenv('model') MODEL = os.getenv("model")
TIMEOUT = int(os.getenv('timeout')) TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv('language').capitalize() LANGUAGE = os.getenv("language").capitalize()
PROMPT = Path('prompt.txt').read_text(encoding='utf-8') PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path('vocab.txt').read_text(encoding='utf-8') VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv('threads')) THREADS = int(os.getenv("threads"))
LOCK = threading.Lock() LOCK = threading.Lock()
WIDTH = int(os.getenv('width')) WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv('listWidth')) LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70 NOTEWIDTH = 70
MAXHISTORY = 10 MAXHISTORY = 10
ESTIMATE = '' ESTIMATE = ""
TOKENS = [0, 0] TOKENS = [0, 0]
NAMESLIST = [] NAMESLIST = []
NAMES = False # Output a list of all the character names found NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = True # Overwrites textwrap FIXTEXTWRAP = True # Overwrites textwrap
IGNORETLTEXT = False # Ignores all translated text. IGNORETLTEXT = False # Ignores all translated text.
MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
#tqdm Globals # tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0 POSITION = 0
LEAVE = False LEAVE = False
# Pricing - Depends on the model https://openai.com/pricing # Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request # Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size. # If you are getting a MISMATCH LENGTH error, lower the batch size.
if 'gpt-3.5' in MODEL: if "gpt-3.5" in MODEL:
INPUTAPICOST = .002 INPUTAPICOST = 0.002
OUTPUTAPICOST = .002 OUTPUTAPICOST = 0.002
BATCHSIZE = 10 BATCHSIZE = 10
elif 'gpt-4' in MODEL: elif "gpt-4" in MODEL:
INPUTAPICOST = .01 INPUTAPICOST = 0.01
OUTPUTAPICOST = .03 OUTPUTAPICOST = 0.03
BATCHSIZE = 50 BATCHSIZE = 50
def handleJSON(filename, estimate): def handleJSON(filename, estimate):
global ESTIMATE, totalTokens global ESTIMATE, totalTokens
ESTIMATE = estimate ESTIMATE = estimate
@ -74,11 +75,11 @@ def handleJSON(filename, estimate):
TOKENS[0] += translatedData[1][0] TOKENS[0] += translatedData[1][0]
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
return getResultString(['', TOKENS, None], end - start, 'TOTAL') return getResultString(["", TOKENS, None], end - start, "TOTAL")
else: else:
try: try:
with open('translated/' + filename, 'w', encoding='UTF-8') as outFile: with open("translated/" + filename, "w", encoding="UTF-8") as outFile:
start = time.time() start = time.time()
translatedData = openFiles(filename) translatedData = openFiles(filename)
@ -90,36 +91,50 @@ def handleJSON(filename, estimate):
TOKENS[0] += translatedData[1][0] TOKENS[0] += translatedData[1][0]
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
except Exception: except Exception:
return 'Fail' return "Fail"
return getResultString(["", TOKENS, None], end - start, "TOTAL")
return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def openFiles(filename): def openFiles(filename):
with open('files/' + filename, 'r', encoding='UTF-8-sig') as f: with open("files/" + filename, "r", encoding="UTF-8-sig") as f:
data = json.load(f) data = json.load(f)
# Map Files # Map Files
if '.json' in filename: if ".json" in filename:
translatedData = parseJSON(data, filename) translatedData = parseJSON(data, filename)
else: else:
raise NameError(filename + ' Not Supported') raise NameError(filename + " Not Supported")
return translatedData return translatedData
def getResultString(translatedData, translationTime, filename): def getResultString(translatedData, translationTime, filename):
# File Print String # File Print String
totalTokenstring =\ totalTokenstring = (
Fore.YELLOW +\ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
'[Input: ' + str(translatedData[1][0]) + ']'\ "[Output: "
'[Output: ' + str(translatedData[1][1]) + ']'\ + str(translatedData[1][1])
'[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\ + "]" "[Cost: ${:,.4f}".format(
(translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']' (translatedData[1][0] * 0.001 * INPUTAPICOST)
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
)
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None: if translatedData[2] == None:
# Success # Success
return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET return (
filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.GREEN
+ " \u2713 "
+ Fore.RESET
)
else: else:
# Fail # Fail
@ -128,18 +143,29 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
errorString = str(e) + Fore.RED errorString = str(e) + Fore.RED
return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\ return (
errorString + Fore.RESET filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.RED
+ " \u2717 "
+ errorString
+ Fore.RESET
)
def parseJSON(data, filename): def parseJSON(data, filename):
totalTokens = [0, 0] totalTokens = [0, 0]
totalLines = 0 totalLines = 0
totalLines = len(data) totalLines = len(data)
global LOCK global LOCK
with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: with tqdm(
pbar.desc=filename bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE
pbar.total=totalLines ) as pbar:
pbar.desc = filename
pbar.total = totalLines
try: try:
result = translateJSON(data, pbar) result = translateJSON(data, pbar)
totalTokens[0] += result[0] totalTokens[0] += result[0]
@ -148,12 +174,13 @@ def parseJSON(data, filename):
return [data, totalTokens, e] return [data, totalTokens, e]
return [data, totalTokens, None] return [data, totalTokens, None]
def translateJSON(data, pbar): def translateJSON(data, pbar):
textHistory = [] textHistory = []
batch = [] batch = []
maxHistory = MAXHISTORY maxHistory = MAXHISTORY
tokens = [0, 0] tokens = [0, 0]
speaker = 'None' speaker = "None"
insertBool = False insertBool = False
i = 0 i = 0
batchStartIndex = 0 batchStartIndex = 0
@ -161,35 +188,43 @@ def translateJSON(data, pbar):
while i < len(data): while i < len(data):
item = data[i] item = data[i]
# Speaker # Speaker
if 'name' in item: if "name" in item:
if item['name'] not in [None, '-']: if item["name"] not in [None, "-"]:
response = getSpeaker(item['name']) response = getSpeaker(item["name"])
speaker = response[0] speaker = response[0]
tokens[0] += response[1][0] tokens[0] += response[1][0]
tokens[1] += response[1][1] tokens[1] += response[1][1]
item['name'] = speaker item["name"] = speaker
else: else:
speaker = 'None' speaker = "None"
pbar.update(1) pbar.update(1)
i += 1 i += 1
# Text # Text
elif 'me' in item: elif "me" in item:
for text in ['text', 'text2', 'help1', 'help2', 'help3', 'like', 'message', 'me']: for text in [
"text",
"text2",
"help1",
"help2",
"help3",
"like",
"message",
"me",
]:
if text in item: if text in item:
if item[text] != None: if item[text] != None:
jaString = item[text] jaString = item[text]
# Remove any textwrap # Remove any textwrap
if FIXTEXTWRAP == True: if FIXTEXTWRAP == True:
finalJAString = jaString.replace('\n', ' ') finalJAString = jaString.replace("\n", " ")
# [Passthrough 1] Pulling From File # [Passthrough 1] Pulling From File
if insertBool is False: if insertBool is False:
# Append to List and Clear Values # Append to List and Clear Values
batch.append(finalJAString) batch.append(finalJAString)
speaker = '' speaker = ""
# Translate Batch if Full # Translate Batch if Full
if len(batch) == BATCHSIZE: if len(batch) == BATCHSIZE:
@ -207,7 +242,7 @@ def translateJSON(data, pbar):
# Mismatch # Mismatch
else: else:
pbar.write(f'Mismatch: {batchStartIndex} - {i}') pbar.write(f"Mismatch: {batchStartIndex} - {i}")
MISMATCH.append(batch) MISMATCH.append(batch)
batchStartIndex = i batchStartIndex = i
batch.clear() batch.clear()
@ -215,7 +250,7 @@ def translateJSON(data, pbar):
if insertBool is False: if insertBool is False:
pbar.update(1) pbar.update(1)
i += 1 i += 1
currentGroup = [] currentGroup = []
# [Passthrough 2] Setting Data # [Passthrough 2] Setting Data
@ -224,15 +259,15 @@ def translateJSON(data, pbar):
translatedText = translatedBatch[0] translatedText = translatedBatch[0]
# Remove added speaker # Remove added speaker
translatedText = re.sub(r'^.+?:\s', '', translatedText) translatedText = re.sub(r"^.+?:\s", "", translatedText)
# Textwrap # Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH) translatedText = textwrap.fill(translatedText, width=WIDTH)
# Set Text # Set Text
item[text] = translatedText item[text] = translatedText
translatedBatch.pop(0) translatedBatch.pop(0)
speaker = '' speaker = ""
currentGroup = [] currentGroup = []
i += 1 i += 1
@ -261,93 +296,99 @@ def translateJSON(data, pbar):
# Mismatch # Mismatch
else: else:
pbar.write(f'Mismatch: {batchStartIndex} - {i}') pbar.write(f"Mismatch: {batchStartIndex} - {i}")
MISMATCH.append(batch) MISMATCH.append(batch)
batchStartIndex = i batchStartIndex = i
batch.clear() batch.clear()
currentGroup = [] currentGroup = []
return tokens return tokens
# Save some money and enter the character before translation # Save some money and enter the character before translation
def getSpeaker(speaker): def getSpeaker(speaker):
match speaker: match speaker:
case 'セレナ': case "セレナ":
return ['Serena', [0,0]] return ["Serena", [0, 0]]
case 'レナ': case "レナ":
return ['Rena', [0,0]] return ["Rena", [0, 0]]
case 'フィルス': case "フィルス":
return ['Phils', [0,0]] return ["Phils", [0, 0]]
case 'レイン': case "レイン":
return ['Meryl', [0,0]] return ["Meryl", [0, 0]]
case _: case _:
return translateGPT(speaker, 'Reply with only the '+ LANGUAGE +' translation of the NPC name.', False) return translateGPT(
speaker,
"Reply with only the " + LANGUAGE + " translation of the NPC name.",
False,
)
def subVars(jaString): def subVars(jaString):
jaString = jaString.replace('\u3000', ' ') jaString = jaString.replace("\u3000", " ")
# Nested # Nested
count = 0 count = 0
nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString) nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList) nestedList = set(nestedList)
if len(nestedList) != 0: if len(nestedList) != 0:
for icon in nestedList: for icon in nestedList:
jaString = jaString.replace(icon, '{Nested_' + str(count) + '}') jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString) iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList) iconList = set(iconList)
if len(iconList) != 0: if len(iconList) != 0:
for icon in iconList: for icon in iconList:
jaString = jaString.replace(icon, '{Ascii_' + str(count) + '}') jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString) colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList) colorList = set(colorList)
if len(colorList) != 0: if len(colorList) != 0:
for color in colorList: for color in colorList:
jaString = jaString.replace(color, '{Color_' + str(count) + '}') jaString = jaString.replace(color, "{Color_" + str(count) + "}")
count += 1 count += 1
# Names # Names
count = 0 count = 0
nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString) nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList) nameList = set(nameList)
if len(nameList) != 0: if len(nameList) != 0:
for name in nameList: for name in nameList:
jaString = jaString.replace(name, '{Noun_' + str(count) + '}') jaString = jaString.replace(name, "{Noun_" + str(count) + "}")
count += 1 count += 1
# Variables # Variables
count = 0 count = 0
varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString) varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList) varList = set(varList)
if len(varList) != 0: if len(varList) != 0:
for var in varList: for var in varList:
jaString = jaString.replace(var, '{Var_' + str(count) + '}') jaString = jaString.replace(var, "{Var_" + str(count) + "}")
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
formatList = re.findall(r'[\\]+[\w]+\[.+?\]', jaString) formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
formatList = set(formatList) formatList = set(formatList)
if len(formatList) != 0: if len(formatList) != 0:
for var in formatList: for var in formatList:
jaString = jaString.replace(var, '{FCode_' + str(count) + '}') jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
count += 1 count += 1
# Put all lists in list and return # Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList] allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList] return [jaString, allList]
def resubVars(translatedText, allList): def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense # Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r'\[\s?.+?\s?\]', translatedText) matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0: if len(matchList) > 0:
for match in matchList: for match in matchList:
text = match.strip() text = match.strip()
@ -357,54 +398,58 @@ def resubVars(translatedText, allList):
count = 0 count = 0
if len(allList[0]) != 0: if len(allList[0]) != 0:
for var in allList[0]: for var in allList[0]:
translatedText = translatedText.replace('{Nested_' + str(count) + '}', var) translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
if len(allList[1]) != 0: if len(allList[1]) != 0:
for var in allList[1]: for var in allList[1]:
translatedText = translatedText.replace('{Ascii_' + str(count) + '}', var) translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
if len(allList[2]) != 0: if len(allList[2]) != 0:
for var in allList[2]: for var in allList[2]:
translatedText = translatedText.replace('{Color_' + str(count) + '}', var) translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
count += 1 count += 1
# Names # Names
count = 0 count = 0
if len(allList[3]) != 0: if len(allList[3]) != 0:
for var in allList[3]: for var in allList[3]:
translatedText = translatedText.replace('{Noun_' + str(count) + '}', var) translatedText = translatedText.replace("{Noun_" + str(count) + "}", var)
count += 1 count += 1
# Vars # Vars
count = 0 count = 0
if len(allList[4]) != 0: if len(allList[4]) != 0:
for var in allList[4]: for var in allList[4]:
translatedText = translatedText.replace('{Var_' + str(count) + '}', var) translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
if len(allList[5]) != 0: if len(allList[5]) != 0:
for var in allList[5]: for var in allList[5]:
translatedText = translatedText.replace('{FCode_' + str(count) + '}', var) translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
count += 1 count += 1
return translatedText return translatedText
def batchList(input_list, batch_size): def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0: if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer") raise ValueError("batch_size must be a positive integer")
return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)] return [
input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
]
def createContext(fullPromptFlag, subbedT): def createContext(fullPromptFlag, subbedT):
characters = 'Game Characters:\n\ characters = "Game Characters:\n\
ルナリア (Lunaria) - Female\n\ ルナリア (Lunaria) - Female\n\
ソニア (Sonia) - Female\n\ ソニア (Sonia) - Female\n\
マナ (Mana) - Female\n\ マナ (Mana) - Female\n\
@ -419,10 +464,12 @@ def createContext(fullPromptFlag, subbedT):
ツキハ (Tsukiha) - Female\n\ ツキハ (Tsukiha) - Female\n\
フィリカ (Filica) - Female\n\ フィリカ (Filica) - Female\n\
レノ (Renno) - Female\n\ レノ (Renno) - Female\n\
' "
system = PROMPT if fullPromptFlag else \ system = (
f"\ PROMPT
if fullPromptFlag
else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
You are going to be translating text from a videogame.\n\ You are going to be translating text from a videogame.\n\
I will give you lines of text, and you must translate each line to the best of your ability.\n\ I will give you lines of text, and you must translate each line to the best of your ability.\n\
@ -439,9 +486,11 @@ I will give you lines of text, and you must translate each line to the best of y
- Translate 'よかった' as 'thank goodness'\n\ - Translate 'よかった' as 'thank goodness'\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\
" "
user = f'{subbedT}' )
user = f"{subbedT}"
return characters, system, user return characters, system, user
def translateText(characters, system, user, history): def translateText(characters, system, user, history):
# Prompt # Prompt
msg = [{"role": "system", "content": system + characters}] msg = [{"role": "system", "content": system + characters}]
@ -454,9 +503,9 @@ def translateText(characters, system, user, history):
msg.extend([{"role": "assistant", "content": h} for h in history]) msg.extend([{"role": "assistant", "content": h} for h in history])
else: else:
msg.append({"role": "assistant", "content": history}) msg.append({"role": "assistant", "content": history})
# Content to TL # Content to TL
msg.append({"role": "user", "content": f'{user}'}) msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create( response = openai.chat.completions.create(
temperature=0.1, temperature=0.1,
frequency_penalty=0.1, frequency_penalty=0.1,
@ -466,37 +515,44 @@ def translateText(characters, system, user, history):
) )
return response return response
def cleanTranslatedText(translatedText, varResponse): def cleanTranslatedText(translatedText, varResponse):
placeholders = { placeholders = {
f'{LANGUAGE} Translation: ': '', f"{LANGUAGE} Translation: ": "",
'Translation: ': '', "Translation: ": "",
'っ': '', "っ": "",
'〜': '~', "〜": "~",
'ッ': '', "ッ": "",
'。': '.', "。": ".",
'Placeholder Text': '' "Placeholder Text": "",
# Add more replacements as needed # Add more replacements as needed
} }
for target, replacement in placeholders.items(): for target, replacement in placeholders.items():
translatedText = translatedText.replace(target, replacement) translatedText = translatedText.replace(target, replacement)
translatedText = resubVars(translatedText, varResponse[1]) translatedText = resubVars(translatedText, varResponse[1])
return [line for line in translatedText.split('\n') if line] return [line for line in translatedText.split("\n") if line]
def extractTranslation(translatedTextList, is_list): def extractTranslation(translatedTextList, is_list):
pattern = r'`?<Line(\d+)>([\\]*.*?[\\]*?)<\/?Line\d+>`?' pattern = r"`?<Line(\d+)>([\\]*.*?[\\]*?)<\/?Line\d+>`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item. # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list: if is_list:
return [re.findall(pattern, line)[0][1] for line in translatedTextList if re.search(pattern, line)] return [
re.findall(pattern, line)[0][1]
for line in translatedTextList
if re.search(pattern, line)
]
else: else:
matchList = re.findall(pattern, translatedTextList) matchList = re.findall(pattern, translatedTextList)
return matchList[0][1] if matchList else translatedTextList return matchList[0][1] if matchList else translatedTextList
def countTokens(characters, system, user, history): def countTokens(characters, system, user, history):
inputTotalTokens = 0 inputTotalTokens = 0
outputTotalTokens = 0 outputTotalTokens = 0
enc = tiktoken.encoding_for_model('gpt-4') enc = tiktoken.encoding_for_model("gpt-4")
# Input # Input
if isinstance(history, list): if isinstance(history, list):
for line in history: for line in history:
@ -508,15 +564,17 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user)) inputTotalTokens += len(enc.encode(user))
# Output # Output
outputTotalTokens += round(len(enc.encode(user))*3) outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens] return [inputTotalTokens, outputTotalTokens]
def combineList(tlist, text): def combineList(tlist, text):
if isinstance(text, list): if isinstance(text, list):
return [t for sublist in tlist for t in sublist] return [t for sublist in tlist for t in sublist]
return tlist[0] return tlist[0]
@retry(exceptions=Exception, tries=5, delay=5) @retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag): def translateGPT(text, history, fullPromptFlag):
totalTokens = [0, 0] totalTokens = [0, 0]
@ -528,8 +586,10 @@ def translateGPT(text, history, fullPromptFlag):
for index, tItem in enumerate(tList): for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting # Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list): if isinstance(tItem, list):
payload = '\n'.join([f'`<Line{i}>{item}</Line{i}>`' for i, item in enumerate(tItem)]) payload = "\n".join(
payload = payload.replace('``', '`Placeholder Text`') [f"`<Line{i}>{item}</Line{i}>`" for i, item in enumerate(tItem)]
)
payload = payload.replace("``", "`Placeholder Text`")
varResponse = subVars(payload) varResponse = subVars(payload)
subbedT = varResponse[0] subbedT = varResponse[0]
else: else:
@ -537,7 +597,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0] subbedT = varResponse[0]
# Things to Check before starting translation # Things to Check before starting translation
if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT): if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue continue
# Create Message # Create Message
@ -562,12 +622,14 @@ def translateGPT(text, history, fullPromptFlag):
extractedTranslations = extractTranslation(translatedTextList, True) extractedTranslations = extractTranslation(translatedTextList, True)
tList[index] = extractedTranslations tList[index] = extractedTranslations
if len(tItem) != len(translatedTextList): if len(tItem) != len(translatedTextList):
mismatch = True # Just here so breakpoint can be set mismatch = True # Just here so breakpoint can be set
history = extractedTranslations[-10:] # Update history if we have a list history = extractedTranslations[-10:] # Update history if we have a list
else: else:
# Ensure we're passing a single string to extractTranslation # Ensure we're passing a single string to extractTranslation
extractedTranslations = extractTranslation('\n'.join(translatedTextList), False) extractedTranslations = extractTranslation(
"\n".join(translatedTextList), False
)
tList[index] = extractedTranslations tList[index] = extractedTranslations
finalList = combineList(tList, text) finalList = combineList(tList, text)
return [finalList, totalTokens] return [finalList, totalTokens]

411
modules/kansen.py
View file

@ -15,49 +15,50 @@ from tqdm import tqdm
# Open AI # Open AI
load_dotenv() load_dotenv()
if os.getenv('api').replace(' ', '') != '': if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv('api') openai.base_url = os.getenv("api")
openai.organization = os.getenv('org') openai.organization = os.getenv("org")
openai.api_key = os.getenv('key') openai.api_key = os.getenv("key")
#Globals # Globals
MODEL = os.getenv('model') MODEL = os.getenv("model")
TIMEOUT = int(os.getenv('timeout')) TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv('language').capitalize() LANGUAGE = os.getenv("language").capitalize()
PROMPT = Path('prompt.txt').read_text(encoding='utf-8') PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path('vocab.txt').read_text(encoding='utf-8') VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv('threads')) THREADS = int(os.getenv("threads"))
LOCK = threading.Lock() LOCK = threading.Lock()
WIDTH = int(os.getenv('width')) WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv('listWidth')) LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70 NOTEWIDTH = 70
MAXHISTORY = 10 MAXHISTORY = 10
ESTIMATE = '' ESTIMATE = ""
TOKENS = [0, 0] TOKENS = [0, 0]
NAMESLIST = [] NAMESLIST = []
NAMES = False # Output a list of all the character names found NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = False # Overwrites textwrap FIXTEXTWRAP = False # Overwrites textwrap
IGNORETLTEXT = False # Ignores all translated text. IGNORETLTEXT = False # Ignores all translated text.
MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
#tqdm Globals # tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0 POSITION = 0
LEAVE = False LEAVE = False
# Pricing - Depends on the model https://openai.com/pricing # Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request # Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size. # If you are getting a MISMATCH LENGTH error, lower the batch size.
if 'gpt-3.5' in MODEL: if "gpt-3.5" in MODEL:
INPUTAPICOST = .002 INPUTAPICOST = 0.002
OUTPUTAPICOST = .002 OUTPUTAPICOST = 0.002
BATCHSIZE = 10 BATCHSIZE = 10
elif 'gpt-4' in MODEL: elif "gpt-4" in MODEL:
INPUTAPICOST = .01 INPUTAPICOST = 0.01
OUTPUTAPICOST = .03 OUTPUTAPICOST = 0.03
BATCHSIZE = 10 BATCHSIZE = 10
def handleKansen(filename, estimate): def handleKansen(filename, estimate):
global ESTIMATE global ESTIMATE
ESTIMATE = estimate ESTIMATE = estimate
@ -74,17 +75,21 @@ def handleKansen(filename, estimate):
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
# Print Total # Print Total
totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL') totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps # Print any errors on maps
if len(MISMATCH) > 0: if len(MISMATCH) > 0:
return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET return (
totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
)
else: else:
return totalString return totalString
else: else:
try: try:
with open('translated/' + filename, 'w', encoding='shift_jis', errors='ignore') as outFile: with open(
"translated/" + filename, "w", encoding="shift_jis", errors="ignore"
) as outFile:
start = time.time() start = time.time()
translatedData = openFiles(filename) translatedData = openFiles(filename)
@ -97,23 +102,36 @@ def handleKansen(filename, estimate):
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
except Exception: except Exception:
traceback.print_exc() traceback.print_exc()
return 'Fail' return "Fail"
return getResultString(["", TOKENS, None], end - start, "TOTAL")
return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def getResultString(translatedData, translationTime, filename): def getResultString(translatedData, translationTime, filename):
# File Print String # File Print String
totalTokenstring =\ totalTokenstring = (
Fore.YELLOW +\ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
'[Input: ' + str(translatedData[1][0]) + ']'\ "[Output: "
'[Output: ' + str(translatedData[1][1]) + ']'\ + str(translatedData[1][1])
'[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\ + "]" "[Cost: ${:,.4f}".format(
(translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']' (translatedData[1][0] * 0.001 * INPUTAPICOST)
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
)
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None: if translatedData[2] == None:
# Success # Success
return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET return (
filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.GREEN
+ " \u2713 "
+ Fore.RESET
)
else: else:
# Fail # Fail
@ -122,33 +140,45 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
errorString = str(e) + Fore.RED errorString = str(e) + Fore.RED
return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\ return (
errorString + Fore.RESET filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.RED
+ " \u2717 "
+ errorString
+ Fore.RESET
)
def openFiles(filename): def openFiles(filename):
with open('files/' + filename, 'r', encoding='cp932') as readFile: with open("files/" + filename, "r", encoding="cp932") as readFile:
translatedData = parseTyrano(readFile, filename) translatedData = parseTyrano(readFile, filename)
# Delete lines marked for deletion # Delete lines marked for deletion
finalData = [] finalData = []
for line in translatedData[0]: for line in translatedData[0]:
if line != '\\d\n': if line != "\\d\n":
finalData.append(line) finalData.append(line)
translatedData[0] = finalData translatedData[0] = finalData
return translatedData return translatedData
def parseTyrano(readFile, filename): def parseTyrano(readFile, filename):
totalTokens = [0,0] totalTokens = [0, 0]
totalLines = 0 totalLines = 0
# Get total for progress bar # Get total for progress bar
data = readFile.readlines() data = readFile.readlines()
totalLines = len(data) totalLines = len(data)
with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: with tqdm(
pbar.desc=filename bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE
pbar.total=totalLines ) as pbar:
pbar.desc = filename
pbar.total = totalLines
try: try:
result = translateTyrano(data, pbar, totalLines) result = translateTyrano(data, pbar, totalLines)
@ -159,13 +189,14 @@ def parseTyrano(readFile, filename):
return [data, totalTokens, e] return [data, totalTokens, e]
return [data, totalTokens, None] return [data, totalTokens, None]
def translateTyrano(data, pbar, totalLines): def translateTyrano(data, pbar, totalLines):
textHistory = [] textHistory = []
batch = [] batch = []
currentGroup = [] currentGroup = []
maxHistory = MAXHISTORY maxHistory = MAXHISTORY
tokens = [0,0] tokens = [0, 0]
speaker = '' speaker = ""
insertBool = False insertBool = False
global LOCK, ESTIMATE global LOCK, ESTIMATE
i = 0 i = 0
@ -173,108 +204,118 @@ def translateTyrano(data, pbar, totalLines):
while i < len(data): while i < len(data):
# Speaker # Speaker
if '[ns]' in data[i]: if "[ns]" in data[i]:
matchList = re.findall(r'\[ns\](.+?)\[', data[i]) matchList = re.findall(r"\[ns\](.+?)\[", data[i])
if len(matchList) != 0: if len(matchList) != 0:
response = getSpeaker(matchList[0]) response = getSpeaker(matchList[0])
speaker = response[0] speaker = response[0]
tokens[0] += response[1][0] tokens[0] += response[1][0]
tokens[1] += response[1][1] tokens[1] += response[1][1]
data[i] = '[ns]' + speaker + '[nse]\n' data[i] = "[ns]" + speaker + "[nse]\n"
else: else:
speaker = '' speaker = ""
# Choices # Choices
elif '[sel' in data[i]: elif "[sel" in data[i]:
matchList = re.findall(r'\[sel.+text="(.+?)".+', data[i]) matchList = re.findall(r'\[sel.+text="(.+?)".+', data[i])
if len(matchList) != 0: if len(matchList) != 0:
originalText = matchList[0] originalText = matchList[0]
if len(textHistory) > 0: if len(textHistory) > 0:
response = translateGPT(matchList[0], 'Keep your translation as brief as possible. Previous text for context: ' + textHistory[len(textHistory)-1] + '\n\nReply in the style of a dialogue option.', False) response = translateGPT(
matchList[0],
"Keep your translation as brief as possible. Previous text for context: "
+ textHistory[len(textHistory) - 1]
+ "\n\nReply in the style of a dialogue option.",
False,
)
else: else:
response = translateGPT(matchList[0], '\n\nReply in the style of a dialogue option.', False) response = translateGPT(
matchList[0],
"\n\nReply in the style of a dialogue option.",
False,
)
translatedText = response[0] translatedText = response[0]
tokens[0] += response[1][0] tokens[0] += response[1][0]
tokens[1] += response[1][1] tokens[1] += response[1][1]
# Remove characters that may break scripts # Remove characters that may break scripts
charList = ['.', '\"', '\\n'] charList = [".", '"', "\\n"]
for char in charList: for char in charList:
translatedText = translatedText.replace(char, '') translatedText = translatedText.replace(char, "")
# Escape all ' # Escape all '
translatedText = translatedText.replace('\\', '') translatedText = translatedText.replace("\\", "")
# translatedText = translatedText.replace("'", "\\\'") # translatedText = translatedText.replace("'", "\\\'")
# Set Data # Set Data
translatedText = data[i].replace(originalText, translatedText) translatedText = data[i].replace(originalText, translatedText)
data[i] = translatedText data[i] = translatedText
# Lines # Lines
matchList = re.findall(r'(.+?)\[[rpcms_sel]+\]$', data[i]) matchList = re.findall(r"(.+?)\[[rpcms_sel]+\]$", data[i])
if len(matchList) > 0: if len(matchList) > 0:
if 'hisout' in matchList[0]: if "hisout" in matchList[0]:
i += 1 i += 1
continue continue
currentGroup.append(matchList[0]) currentGroup.append(matchList[0])
if len(data) > i+1: if len(data) > i + 1:
while '[r]' in data[i+1]: while "[r]" in data[i + 1]:
if insertBool is True: if insertBool is True:
data[i] = r'\d\n' data[i] = r"\d\n"
pbar.update(1) pbar.update(1)
i += 1 i += 1
matchList = re.findall(r'(.+?)\[r\]', data[i]) matchList = re.findall(r"(.+?)\[r\]", data[i])
if len(matchList) > 0: if len(matchList) > 0:
currentGroup.append(matchList[0]) currentGroup.append(matchList[0])
while '[pcms]' in data[i+1]: while "[pcms]" in data[i + 1]:
if insertBool is True: if insertBool is True:
data[i] = r'\d\n' data[i] = r"\d\n"
pbar.update(1) pbar.update(1)
i += 1 i += 1
matchList = re.findall(r'(.+?)\[pcms\]', data[i]) matchList = re.findall(r"(.+?)\[pcms\]", data[i])
if len(matchList) > 0: if len(matchList) > 0:
currentGroup.append(matchList[0]) currentGroup.append(matchList[0])
while '[pcms_sel]' in data[i+1]: while "[pcms_sel]" in data[i + 1]:
if insertBool is True: if insertBool is True:
data[i] = r'\d\n' data[i] = r"\d\n"
pbar.update(1) pbar.update(1)
i += 1 i += 1
matchList = re.findall(r'(.+?)\[pcms_sel\]', data[i]) matchList = re.findall(r"(.+?)\[pcms_sel\]", data[i])
if len(matchList) > 0: if len(matchList) > 0:
currentGroup.append(matchList[0]) currentGroup.append(matchList[0])
# Join up 401 groups for better translation. # Join up 401 groups for better translation.
if len(currentGroup) > 0: if len(currentGroup) > 0:
finalJAString = ' '.join(currentGroup) finalJAString = " ".join(currentGroup)
oldjaString = finalJAString oldjaString = finalJAString
# Remove any textwrap # Remove any textwrap
if FIXTEXTWRAP == True: if FIXTEXTWRAP == True:
finalJAString = finalJAString.replace('[r]', ' ') finalJAString = finalJAString.replace("[r]", " ")
# Remove Extra Stuff bad for translation. # Remove Extra Stuff bad for translation.
finalJAString = finalJAString.replace('ﾞ', '') finalJAString = finalJAString.replace("ﾞ", "")
finalJAString = finalJAString.replace('・', '.') finalJAString = finalJAString.replace("・", ".")
finalJAString = finalJAString.replace('‶', '') finalJAString = finalJAString.replace("‶", "")
finalJAString = finalJAString.replace('”', '') finalJAString = finalJAString.replace("”", "")
finalJAString = finalJAString.replace('―', '-') finalJAString = finalJAString.replace("―", "-")
finalJAString = finalJAString.replace('…', '...') finalJAString = finalJAString.replace("…", "...")
finalJAString = re.sub(r'(\.{3}\.+)', '...', finalJAString) finalJAString = re.sub(r"(\.{3}\.+)", "...", finalJAString)
finalJAString = finalJAString.replace('　', ' ') finalJAString = finalJAString.replace("　", " ")
# Furigana Removal # Furigana Removal
matchList = re.findall(r'(\[ruby\stext=.+text=\"(.+)\"\])', finalJAString) matchList = re.findall(r"(\[ruby\stext=.+text=\"(.+)\"\])", finalJAString)
if len(matchList) > 0: if len(matchList) > 0:
finalJAString = finalJAString.replace(matchList[0][0], matchList[0][1]) finalJAString = finalJAString.replace(matchList[0][0], matchList[0][1])
# Add Speaker (If there is one) # Add Speaker (If there is one)
if speaker != '': if speaker != "":
finalJAString = f'{speaker}: {finalJAString}' finalJAString = f"{speaker}: {finalJAString}"
# [Passthrough 1] Pulling From File # [Passthrough 1] Pulling From File
if insertBool is False: if insertBool is False:
# Append to List and Clear Values # Append to List and Clear Values
batch.append(finalJAString) batch.append(finalJAString)
speaker = '' speaker = ""
# Translate Batch if Full # Translate Batch if Full
if len(batch) == BATCHSIZE: if len(batch) == BATCHSIZE:
@ -292,7 +333,7 @@ def translateTyrano(data, pbar, totalLines):
# Mismatch # Mismatch
else: else:
pbar.write(f'Mismatch: {batchStartIndex} - {i}') pbar.write(f"Mismatch: {batchStartIndex} - {i}")
MISMATCH.append(batch) MISMATCH.append(batch)
batchStartIndex = i batchStartIndex = i
batch.clear() batch.clear()
@ -306,31 +347,31 @@ def translateTyrano(data, pbar, totalLines):
else: else:
# Get Text # Get Text
translatedText = translatedBatch[0] translatedText = translatedBatch[0]
translatedText = translatedText.replace('\\"', '\"') translatedText = translatedText.replace('\\"', '"')
translatedText = translatedText.replace('[', '(') translatedText = translatedText.replace("[", "(")
translatedText = translatedText.replace(']', ')') translatedText = translatedText.replace("]", ")")
# Remove added speaker # Remove added speaker
translatedText = re.sub(r'^.+?:\s', '', translatedText) translatedText = re.sub(r"^.+?:\s", "", translatedText)
# Textwrap # Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH) translatedText = textwrap.fill(translatedText, width=WIDTH)
textList = translatedText.split('\n') textList = translatedText.split("\n")
# Set Text # Set Text
data[i] = r'\d\n' data[i] = r"\d\n"
for line in textList: for line in textList:
# Wordwrap Text # Wordwrap Text
if '[r]' not in line: if "[r]" not in line:
line = textwrap.fill(line, width=WIDTH) line = textwrap.fill(line, width=WIDTH)
line = line.replace('\n', '[r]') line = line.replace("\n", "[r]")
# Set # Set
data.insert(i, line.strip() + '[r]\n') data.insert(i, line.strip() + "[r]\n")
i+=1 i += 1
data[i-1] = data[i-1].replace('[r]', '[pcms]') data[i - 1] = data[i - 1].replace("[r]", "[pcms]")
translatedBatch.pop(0) translatedBatch.pop(0)
speaker = '' speaker = ""
currentGroup = [] currentGroup = []
# If Batch is empty. Move on. # If Batch is empty. Move on.
@ -361,7 +402,7 @@ def translateTyrano(data, pbar, totalLines):
# Mismatch # Mismatch
else: else:
pbar.write(f'Mismatch: {batchStartIndex} - {i}') pbar.write(f"Mismatch: {batchStartIndex} - {i}")
MISMATCH.append(batch) MISMATCH.append(batch)
batchStartIndex = i batchStartIndex = i
batch.clear() batch.clear()
@ -369,92 +410,99 @@ def translateTyrano(data, pbar, totalLines):
currentGroup = [] currentGroup = []
return tokens return tokens
# Save some money and enter the character before translation # Save some money and enter the character before translation
def getSpeaker(speaker): def getSpeaker(speaker):
match speaker: match speaker:
case '央': case "央":
return ['Akira', [0,0]] return ["Akira", [0, 0]]
case '累': case "累":
return ['Rui', [0,0]] return ["Rui", [0, 0]]
case '梨里': case "梨里":
return ['Riri', [0,0]] return ["Riri", [0, 0]]
case '純': case "純":
return ['Jun', [0,0]] return ["Jun", [0, 0]]
case '美鈴': case "美鈴":
return ['Misuzu', [0,0]] return ["Misuzu", [0, 0]]
case '須田': case "須田":
return ['Suda', [0,0]] return ["Suda", [0, 0]]
case '高橋': case "高橋":
return ['Takahashi', [0,0]] return ["Takahashi", [0, 0]]
case '勇二': case "勇二":
return ['Yuuji', [0,0]] return ["Yuuji", [0, 0]]
case _: case _:
return translateGPT(speaker, 'Reply with only the '+ LANGUAGE +' translation of the NPC name.', False) return translateGPT(
speaker,
"Reply with only the " + LANGUAGE + " translation of the NPC name.",
False,
)
def subVars(jaString): def subVars(jaString):
jaString = jaString.replace('\u3000', ' ') jaString = jaString.replace("\u3000", " ")
# Nested # Nested
count = 0 count = 0
nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString) nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList) nestedList = set(nestedList)
if len(nestedList) != 0: if len(nestedList) != 0:
for icon in nestedList: for icon in nestedList:
jaString = jaString.replace(icon, '{Nested_' + str(count) + '}') jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString) iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList) iconList = set(iconList)
if len(iconList) != 0: if len(iconList) != 0:
for icon in iconList: for icon in iconList:
jaString = jaString.replace(icon, '{Ascii_' + str(count) + '}') jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString) colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList) colorList = set(colorList)
if len(colorList) != 0: if len(colorList) != 0:
for color in colorList: for color in colorList:
jaString = jaString.replace(color, '{Color_' + str(count) + '}') jaString = jaString.replace(color, "{Color_" + str(count) + "}")
count += 1 count += 1
# Names # Names
count = 0 count = 0
nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString) nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList) nameList = set(nameList)
if len(nameList) != 0: if len(nameList) != 0:
for name in nameList: for name in nameList:
jaString = jaString.replace(name, '{Noun_' + str(count) + '}') jaString = jaString.replace(name, "{Noun_" + str(count) + "}")
count += 1 count += 1
# Variables # Variables
count = 0 count = 0
varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString) varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList) varList = set(varList)
if len(varList) != 0: if len(varList) != 0:
for var in varList: for var in varList:
jaString = jaString.replace(var, '{Var_' + str(count) + '}') jaString = jaString.replace(var, "{Var_" + str(count) + "}")
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
formatList = re.findall(r'[\\]+[\w]+\[.+?\]', jaString) formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
formatList = set(formatList) formatList = set(formatList)
if len(formatList) != 0: if len(formatList) != 0:
for var in formatList: for var in formatList:
jaString = jaString.replace(var, '{FCode_' + str(count) + '}') jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
count += 1 count += 1
# Put all lists in list and return # Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList] allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList] return [jaString, allList]
def resubVars(translatedText, allList): def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense # Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r'\[\s?.+?\s?\]', translatedText) matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0: if len(matchList) > 0:
for match in matchList: for match in matchList:
text = match.strip() text = match.strip()
@ -464,54 +512,58 @@ def resubVars(translatedText, allList):
count = 0 count = 0
if len(allList[0]) != 0: if len(allList[0]) != 0:
for var in allList[0]: for var in allList[0]:
translatedText = translatedText.replace('{Nested_' + str(count) + '}', var) translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
if len(allList[1]) != 0: if len(allList[1]) != 0:
for var in allList[1]: for var in allList[1]:
translatedText = translatedText.replace('{Ascii_' + str(count) + '}', var) translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
if len(allList[2]) != 0: if len(allList[2]) != 0:
for var in allList[2]: for var in allList[2]:
translatedText = translatedText.replace('{Color_' + str(count) + '}', var) translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
count += 1 count += 1
# Names # Names
count = 0 count = 0
if len(allList[3]) != 0: if len(allList[3]) != 0:
for var in allList[3]: for var in allList[3]:
translatedText = translatedText.replace('{Noun_' + str(count) + '}', var) translatedText = translatedText.replace("{Noun_" + str(count) + "}", var)
count += 1 count += 1
# Vars # Vars
count = 0 count = 0
if len(allList[4]) != 0: if len(allList[4]) != 0:
for var in allList[4]: for var in allList[4]:
translatedText = translatedText.replace('{Var_' + str(count) + '}', var) translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
if len(allList[5]) != 0: if len(allList[5]) != 0:
for var in allList[5]: for var in allList[5]:
translatedText = translatedText.replace('{FCode_' + str(count) + '}', var) translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
count += 1 count += 1
return translatedText return translatedText
def batchList(input_list, batch_size): def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0: if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer") raise ValueError("batch_size must be a positive integer")
return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)] return [
input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
]
def createContext(fullPromptFlag, subbedT): def createContext(fullPromptFlag, subbedT):
characters = 'Game Characters:\n\ characters = "Game Characters:\n\
渋江 央 (Shibue Akira) - Male\n\ 渋江 央 (Shibue Akira) - Male\n\
蘆名 累 (Ashina Rui) - Female\n\ 蘆名 累 (Ashina Rui) - Female\n\
清原 梨里 (Kiyohara Riri) - Female\n\ 清原 梨里 (Kiyohara Riri) - Female\n\
@ -520,19 +572,23 @@ def createContext(fullPromptFlag, subbedT):
須田 (Suda) - Male\n\ 須田 (Suda) - Male\n\
高橋 (Takahashi) - Female\n\ 高橋 (Takahashi) - Female\n\
勇二 (Yuuji) - Male\n\ 勇二 (Yuuji) - Male\n\
' "
system = PROMPT + VOCAB if fullPromptFlag else \ system = (
f"\ PROMPT + VOCAB
if fullPromptFlag
else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
You are going to be translating text from a videogame.\n\ You are going to be translating text from a videogame.\n\
I will give you lines of text, and you must translate each line to the best of your ability.\n\ I will give you lines of text, and you must translate each line to the best of your ability.\n\
{VOCAB}\n\ {VOCAB}\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\
" "
user = f'{subbedT}' )
user = f"{subbedT}"
return characters, system, user return characters, system, user
def translateText(characters, system, user, history): def translateText(characters, system, user, history):
# Prompt # Prompt
msg = [{"role": "system", "content": system + characters}] msg = [{"role": "system", "content": system + characters}]
@ -545,9 +601,9 @@ def translateText(characters, system, user, history):
msg.extend([{"role": "system", "content": h} for h in history]) msg.extend([{"role": "system", "content": h} for h in history])
else: else:
msg.append({"role": "system", "content": history}) msg.append({"role": "system", "content": history})
# Content to TL # Content to TL
msg.append({"role": "user", "content": f'{user}'}) msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create( response = openai.chat.completions.create(
temperature=0.1, temperature=0.1,
frequency_penalty=0.1, frequency_penalty=0.1,
@ -557,37 +613,44 @@ def translateText(characters, system, user, history):
) )
return response return response
def cleanTranslatedText(translatedText, varResponse): def cleanTranslatedText(translatedText, varResponse):
placeholders = { placeholders = {
f'{LANGUAGE} Translation: ': '', f"{LANGUAGE} Translation: ": "",
'Translation: ': '', "Translation: ": "",
'っ': '', "っ": "",
'〜': '~', "〜": "~",
'ッ': '', "ッ": "",
'。': '.', "。": ".",
'Placeholder Text': '' "Placeholder Text": "",
# Add more replacements as needed # Add more replacements as needed
} }
for target, replacement in placeholders.items(): for target, replacement in placeholders.items():
translatedText = translatedText.replace(target, replacement) translatedText = translatedText.replace(target, replacement)
translatedText = resubVars(translatedText, varResponse[1]) translatedText = resubVars(translatedText, varResponse[1])
return [line for line in translatedText.replace('\\n', '\n').split('\n') if line] return [line for line in translatedText.replace("\\n", "\n").split("\n") if line]
def extractTranslation(translatedTextList, is_list): def extractTranslation(translatedTextList, is_list):
pattern = r'`?<Line(\d+)>([\\]*.*?[\\]*?)<\/?Line\d+>`?' pattern = r"`?<Line(\d+)>([\\]*.*?[\\]*?)<\/?Line\d+>`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item. # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list: if is_list:
return [re.findall(pattern, line)[0][1] for line in translatedTextList if re.search(pattern, line)] return [
re.findall(pattern, line)[0][1]
for line in translatedTextList
if re.search(pattern, line)
]
else: else:
matchList = re.findall(pattern, translatedTextList) matchList = re.findall(pattern, translatedTextList)
return matchList[0][1] if matchList else translatedTextList return matchList[0][1] if matchList else translatedTextList
def countTokens(characters, system, user, history): def countTokens(characters, system, user, history):
inputTotalTokens = 0 inputTotalTokens = 0
outputTotalTokens = 0 outputTotalTokens = 0
enc = tiktoken.encoding_for_model('gpt-4') enc = tiktoken.encoding_for_model("gpt-4")
# Input # Input
if isinstance(history, list): if isinstance(history, list):
for line in history: for line in history:
@ -599,15 +662,17 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user)) inputTotalTokens += len(enc.encode(user))
# Output # Output
outputTotalTokens += round(len(enc.encode(user))*3) outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens] return [inputTotalTokens, outputTotalTokens]
def combineList(tlist, text): def combineList(tlist, text):
if isinstance(text, list): if isinstance(text, list):
return [t for sublist in tlist for t in sublist] return [t for sublist in tlist for t in sublist]
return tlist[0] return tlist[0]
@retry(exceptions=Exception, tries=5, delay=5) @retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag): def translateGPT(text, history, fullPromptFlag):
totalTokens = [0, 0] totalTokens = [0, 0]
@ -619,8 +684,10 @@ def translateGPT(text, history, fullPromptFlag):
for index, tItem in enumerate(tList): for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting # Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list): if isinstance(tItem, list):
payload = '\n'.join([f'`<Line{i}>{item}</Line{i}>`' for i, item in enumerate(tItem)]) payload = "\n".join(
payload = payload.replace('``', '`Placeholder Text`') [f"`<Line{i}>{item}</Line{i}>`" for i, item in enumerate(tItem)]
)
payload = payload.replace("``", "`Placeholder Text`")
varResponse = subVars(payload) varResponse = subVars(payload)
subbedT = varResponse[0] subbedT = varResponse[0]
else: else:
@ -628,7 +695,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0] subbedT = varResponse[0]
# Things to Check before starting translation # Things to Check before starting translation
if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT): if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue continue
# Create Message # Create Message
@ -653,11 +720,13 @@ def translateGPT(text, history, fullPromptFlag):
extractedTranslations = extractTranslation(translatedTextList, True) extractedTranslations = extractTranslation(translatedTextList, True)
tList[index] = extractedTranslations tList[index] = extractedTranslations
if len(tItem) != len(translatedTextList): if len(tItem) != len(translatedTextList):
mismatch = True # Just here so breakpoint can be set mismatch = True # Just here so breakpoint can be set
history = extractedTranslations[-10:] # Update history if we have a list history = extractedTranslations[-10:] # Update history if we have a list
else: else:
# Ensure we're passing a single string to extractTranslation # Ensure we're passing a single string to extractTranslation
extractedTranslations = extractTranslation('\n'.join(translatedTextList), False) extractedTranslations = extractTranslation(
"\n".join(translatedTextList), False
)
tList[index] = extractedTranslations tList[index] = extractedTranslations
finalList = combineList(tList, text) finalList = combineList(tList, text)

318
modules/lune.py
View file

@ -16,49 +16,50 @@ from tqdm import tqdm
# Open AI # Open AI
load_dotenv() load_dotenv()
if os.getenv('api').replace(' ', '') != '': if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv('api') openai.base_url = os.getenv("api")
openai.organization = os.getenv('org') openai.organization = os.getenv("org")
openai.api_key = os.getenv('key') openai.api_key = os.getenv("key")
#Globals # Globals
MODEL = os.getenv('model') MODEL = os.getenv("model")
TIMEOUT = int(os.getenv('timeout')) TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv('language').capitalize() LANGUAGE = os.getenv("language").capitalize()
PROMPT = Path('prompt.txt').read_text(encoding='utf-8') PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path('vocab.txt').read_text(encoding='utf-8') VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv('threads')) THREADS = int(os.getenv("threads"))
LOCK = threading.Lock() LOCK = threading.Lock()
WIDTH = int(os.getenv('width')) WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv('listWidth')) LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70 NOTEWIDTH = 70
MAXHISTORY = 10 MAXHISTORY = 10
ESTIMATE = '' ESTIMATE = ""
TOKENS = [0, 0] TOKENS = [0, 0]
NAMESLIST = [] NAMESLIST = []
NAMES = False # Output a list of all the character names found NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = True # Overwrites textwrap FIXTEXTWRAP = True # Overwrites textwrap
IGNORETLTEXT = False # Ignores all translated text. IGNORETLTEXT = False # Ignores all translated text.
MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
#tqdm Globals # tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0 POSITION = 0
LEAVE = False LEAVE = False
# Pricing - Depends on the model https://openai.com/pricing # Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request # Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size. # If you are getting a MISMATCH LENGTH error, lower the batch size.
if 'gpt-3.5' in MODEL: if "gpt-3.5" in MODEL:
INPUTAPICOST = .002 INPUTAPICOST = 0.002
OUTPUTAPICOST = .002 OUTPUTAPICOST = 0.002
BATCHSIZE = 10 BATCHSIZE = 10
elif 'gpt-4' in MODEL: elif "gpt-4" in MODEL:
INPUTAPICOST = .01 INPUTAPICOST = 0.01
OUTPUTAPICOST = .03 OUTPUTAPICOST = 0.03
BATCHSIZE = 50 BATCHSIZE = 50
def handleLune(filename, estimate): def handleLune(filename, estimate):
global ESTIMATE, totalTokens global ESTIMATE, totalTokens
ESTIMATE = estimate ESTIMATE = estimate
@ -74,11 +75,11 @@ def handleLune(filename, estimate):
TOKENS[0] += translatedData[1][0] TOKENS[0] += translatedData[1][0]
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
return getResultString(['', TOKENS, None], end - start, 'TOTAL') return getResultString(["", TOKENS, None], end - start, "TOTAL")
else: else:
try: try:
with open('translated/' + filename, 'w', encoding='UTF-8') as outFile: with open("translated/" + filename, "w", encoding="UTF-8") as outFile:
start = time.time() start = time.time()
translatedData = openFiles(filename) translatedData = openFiles(filename)
@ -90,36 +91,50 @@ def handleLune(filename, estimate):
TOKENS[0] += translatedData[1][0] TOKENS[0] += translatedData[1][0]
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
except Exception: except Exception:
return 'Fail' return "Fail"
return getResultString(["", TOKENS, None], end - start, "TOTAL")
return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def openFiles(filename): def openFiles(filename):
with open('files/' + filename, 'r', encoding='UTF-8-sig') as f: with open("files/" + filename, "r", encoding="UTF-8-sig") as f:
data = json.load(f) data = json.load(f)
# Map Files # Map Files
if '.json' in filename: if ".json" in filename:
translatedData = parseJSON(data, filename) translatedData = parseJSON(data, filename)
else: else:
raise NameError(filename + ' Not Supported') raise NameError(filename + " Not Supported")
return translatedData return translatedData
def getResultString(translatedData, translationTime, filename): def getResultString(translatedData, translationTime, filename):
# File Print String # File Print String
totalTokenstring =\ totalTokenstring = (
Fore.YELLOW +\ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
'[Input: ' + str(translatedData[1][0]) + ']'\ "[Output: "
'[Output: ' + str(translatedData[1][1]) + ']'\ + str(translatedData[1][1])
'[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\ + "]" "[Cost: ${:,.4f}".format(
(translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']' (translatedData[1][0] * 0.001 * INPUTAPICOST)
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
)
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None: if translatedData[2] == None:
# Success # Success
return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET return (
filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.GREEN
+ " \u2713 "
+ Fore.RESET
)
else: else:
# Fail # Fail
@ -128,18 +143,29 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
errorString = str(e) + Fore.RED errorString = str(e) + Fore.RED
return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\ return (
errorString + Fore.RESET filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.RED
+ " \u2717 "
+ errorString
+ Fore.RESET
)
def parseJSON(data, filename): def parseJSON(data, filename):
totalTokens = [0, 0] totalTokens = [0, 0]
totalLines = 0 totalLines = 0
totalLines = len(data) totalLines = len(data)
global LOCK global LOCK
with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: with tqdm(
pbar.desc=filename bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE
pbar.total=totalLines ) as pbar:
pbar.desc = filename
pbar.total = totalLines
try: try:
result = translateJSON(data, pbar) result = translateJSON(data, pbar)
totalTokens[0] += result[0] totalTokens[0] += result[0]
@ -148,12 +174,13 @@ def parseJSON(data, filename):
return [data, totalTokens, e] return [data, totalTokens, e]
return [data, totalTokens, None] return [data, totalTokens, None]
def translateJSON(data, pbar): def translateJSON(data, pbar):
textHistory = [] textHistory = []
batch = [] batch = []
maxHistory = MAXHISTORY maxHistory = MAXHISTORY
tokens = [0, 0] tokens = [0, 0]
speaker = 'None' speaker = "None"
insertBool = False insertBool = False
i = 0 i = 0
batchStartIndex = 0 batchStartIndex = 0
@ -161,32 +188,41 @@ def translateJSON(data, pbar):
while i < len(data): while i < len(data):
item = data[i] item = data[i]
# Speaker # Speaker
if 'name' in item: if "name" in item:
if item['name'] not in [None, '-']: if item["name"] not in [None, "-"]:
response = getSpeaker(item['name']) response = getSpeaker(item["name"])
speaker = response[0] speaker = response[0]
tokens[0] += response[1][0] tokens[0] += response[1][0]
tokens[1] += response[1][1] tokens[1] += response[1][1]
item['name'] = speaker item["name"] = speaker
else: else:
speaker = 'None' speaker = "None"
# Text # Text
if 'message' in item: if "message" in item:
for text in ['text', 'text2', 'help1', 'help2', 'help3', 'like', 'message', 'me']: for text in [
"text",
"text2",
"help1",
"help2",
"help3",
"like",
"message",
"me",
]:
if text in item: if text in item:
if item[text] != None: if item[text] != None:
jaString = item[text] jaString = item[text]
# Remove any textwrap # Remove any textwrap
if FIXTEXTWRAP == True: if FIXTEXTWRAP == True:
finalJAString = jaString.replace('\n', ' ') finalJAString = jaString.replace("\n", " ")
# [Passthrough 1] Pulling From File # [Passthrough 1] Pulling From File
if insertBool is False: if insertBool is False:
# Append to List and Clear Values # Append to List and Clear Values
batch.append(finalJAString) batch.append(finalJAString)
speaker = '' speaker = ""
# Translate Batch if Full # Translate Batch if Full
if len(batch) == BATCHSIZE: if len(batch) == BATCHSIZE:
@ -204,7 +240,7 @@ def translateJSON(data, pbar):
# Mismatch # Mismatch
else: else:
pbar.write(f'Mismatch: {batchStartIndex} - {i}') pbar.write(f"Mismatch: {batchStartIndex} - {i}")
MISMATCH.append(batch) MISMATCH.append(batch)
batchStartIndex = i batchStartIndex = i
batch.clear() batch.clear()
@ -212,7 +248,7 @@ def translateJSON(data, pbar):
if insertBool is False: if insertBool is False:
pbar.update(1) pbar.update(1)
i += 1 i += 1
currentGroup = [] currentGroup = []
# [Passthrough 2] Setting Data # [Passthrough 2] Setting Data
@ -221,15 +257,15 @@ def translateJSON(data, pbar):
translatedText = translatedBatch[0] translatedText = translatedBatch[0]
# Remove added speaker # Remove added speaker
translatedText = re.sub(r'^.+?:\s', '', translatedText) translatedText = re.sub(r"^.+?:\s", "", translatedText)
# Textwrap # Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH) translatedText = textwrap.fill(translatedText, width=WIDTH)
# Set Text # Set Text
item[text] = translatedText item[text] = translatedText
translatedBatch.pop(0) translatedBatch.pop(0)
speaker = '' speaker = ""
currentGroup = [] currentGroup = []
i += 1 i += 1
@ -258,92 +294,99 @@ def translateJSON(data, pbar):
# Mismatch # Mismatch
else: else:
pbar.write(f'Mismatch: {batchStartIndex} - {i}') pbar.write(f"Mismatch: {batchStartIndex} - {i}")
MISMATCH.append(batch) MISMATCH.append(batch)
batchStartIndex = i batchStartIndex = i
batch.clear() batch.clear()
currentGroup = [] currentGroup = []
return tokens return tokens
# Save some money and enter the character before translation # Save some money and enter the character before translation
def getSpeaker(speaker): def getSpeaker(speaker):
match speaker: match speaker:
case 'セレナ': case "セレナ":
return ['Serena', [0,0]] return ["Serena", [0, 0]]
case 'レナ': case "レナ":
return ['Rena', [0,0]] return ["Rena", [0, 0]]
case 'フィルス': case "フィルス":
return ['Phils', [0,0]] return ["Phils", [0, 0]]
case 'レイン': case "レイン":
return ['Meryl', [0,0]] return ["Meryl", [0, 0]]
case _: case _:
return translateGPT(speaker, 'Reply with only the '+ LANGUAGE +' translation of the NPC name.', False) return translateGPT(
speaker,
"Reply with only the " + LANGUAGE + " translation of the NPC name.",
False,
)
def subVars(jaString): def subVars(jaString):
jaString = jaString.replace('\u3000', ' ') jaString = jaString.replace("\u3000", " ")
# Nested # Nested
count = 0 count = 0
nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString) nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList) nestedList = set(nestedList)
if len(nestedList) != 0: if len(nestedList) != 0:
for icon in nestedList: for icon in nestedList:
jaString = jaString.replace(icon, '{Nested_' + str(count) + '}') jaString = jaString.replace(icon, "{Nested_" + str(count) + "}")
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString) iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList) iconList = set(iconList)
if len(iconList) != 0: if len(iconList) != 0:
for icon in iconList: for icon in iconList:
jaString = jaString.replace(icon, '{Ascii_' + str(count) + '}') jaString = jaString.replace(icon, "{Ascii_" + str(count) + "}")
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString) colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList) colorList = set(colorList)
if len(colorList) != 0: if len(colorList) != 0:
for color in colorList: for color in colorList:
jaString = jaString.replace(color, '{Color_' + str(count) + '}') jaString = jaString.replace(color, "{Color_" + str(count) + "}")
count += 1 count += 1
# Names # Names
count = 0 count = 0
nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString) nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList) nameList = set(nameList)
if len(nameList) != 0: if len(nameList) != 0:
for name in nameList: for name in nameList:
jaString = jaString.replace(name, '{Noun_' + str(count) + '}') jaString = jaString.replace(name, "{Noun_" + str(count) + "}")
count += 1 count += 1
# Variables # Variables
count = 0 count = 0
varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString) varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList) varList = set(varList)
if len(varList) != 0: if len(varList) != 0:
for var in varList: for var in varList:
jaString = jaString.replace(var, '{Var_' + str(count) + '}') jaString = jaString.replace(var, "{Var_" + str(count) + "}")
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
formatList = re.findall(r'[\\]+[\w]+\[.+?\]', jaString) formatList = re.findall(r"[\\]+[\w]+\[.+?\]", jaString)
formatList = set(formatList) formatList = set(formatList)
if len(formatList) != 0: if len(formatList) != 0:
for var in formatList: for var in formatList:
jaString = jaString.replace(var, '{FCode_' + str(count) + '}') jaString = jaString.replace(var, "{FCode_" + str(count) + "}")
count += 1 count += 1
# Put all lists in list and return # Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList] allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList] return [jaString, allList]
def resubVars(translatedText, allList): def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense # Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r'\[\s?.+?\s?\]', translatedText) matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0: if len(matchList) > 0:
for match in matchList: for match in matchList:
text = match.strip() text = match.strip()
@ -353,54 +396,58 @@ def resubVars(translatedText, allList):
count = 0 count = 0
if len(allList[0]) != 0: if len(allList[0]) != 0:
for var in allList[0]: for var in allList[0]:
translatedText = translatedText.replace('{Nested_' + str(count) + '}', var) translatedText = translatedText.replace("{Nested_" + str(count) + "}", var)
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
if len(allList[1]) != 0: if len(allList[1]) != 0:
for var in allList[1]: for var in allList[1]:
translatedText = translatedText.replace('{Ascii_' + str(count) + '}', var) translatedText = translatedText.replace("{Ascii_" + str(count) + "}", var)
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
if len(allList[2]) != 0: if len(allList[2]) != 0:
for var in allList[2]: for var in allList[2]:
translatedText = translatedText.replace('{Color_' + str(count) + '}', var) translatedText = translatedText.replace("{Color_" + str(count) + "}", var)
count += 1 count += 1
# Names # Names
count = 0 count = 0
if len(allList[3]) != 0: if len(allList[3]) != 0:
for var in allList[3]: for var in allList[3]:
translatedText = translatedText.replace('{Noun_' + str(count) + '}', var) translatedText = translatedText.replace("{Noun_" + str(count) + "}", var)
count += 1 count += 1
# Vars # Vars
count = 0 count = 0
if len(allList[4]) != 0: if len(allList[4]) != 0:
for var in allList[4]: for var in allList[4]:
translatedText = translatedText.replace('{Var_' + str(count) + '}', var) translatedText = translatedText.replace("{Var_" + str(count) + "}", var)
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
if len(allList[5]) != 0: if len(allList[5]) != 0:
for var in allList[5]: for var in allList[5]:
translatedText = translatedText.replace('{FCode_' + str(count) + '}', var) translatedText = translatedText.replace("{FCode_" + str(count) + "}", var)
count += 1 count += 1
return translatedText return translatedText
def batchList(input_list, batch_size): def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0: if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer") raise ValueError("batch_size must be a positive integer")
return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)] return [
input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
]
def createContext(fullPromptFlag, subbedT): def createContext(fullPromptFlag, subbedT):
characters = 'Game Characters:\n\ characters = "Game Characters:\n\
林つかさ (Tsukasa Hayashi) - Female\n\ 林つかさ (Tsukasa Hayashi) - Female\n\
山田美兎 (Miyato Yamada) - Female\n\ 山田美兎 (Miyato Yamada) - Female\n\
鈴木赤音 (Akane Suzuki) - Female\n\ 鈴木赤音 (Akane Suzuki) - Female\n\
@ -413,13 +460,17 @@ def createContext(fullPromptFlag, subbedT):
モリー・ボイド (Molly Boyd) - Female\n\ モリー・ボイド (Molly Boyd) - Female\n\
オルガ・ブヤチッチ (Olga Buyachich) - Female\n\ オルガ・ブヤチッチ (Olga Buyachich) - Female\n\
アッチャラー ギッティ (Atchara Gitti) - Female\n\ アッチャラー ギッティ (Atchara Gitti) - Female\n\
' "
system = PROMPT if fullPromptFlag else \ system = (
f'Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`' PROMPT
user = f'{subbedT}' if fullPromptFlag
else f"Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`"
)
user = f"{subbedT}"
return characters, system, user return characters, system, user
def translateText(characters, system, user, history): def translateText(characters, system, user, history):
# Prompt # Prompt
msg = [{"role": "system", "content": system + characters}] msg = [{"role": "system", "content": system + characters}]
@ -432,9 +483,9 @@ def translateText(characters, system, user, history):
msg.extend([{"role": "assistant", "content": h} for h in history]) msg.extend([{"role": "assistant", "content": h} for h in history])
else: else:
msg.append({"role": "assistant", "content": history}) msg.append({"role": "assistant", "content": history})
# Content to TL # Content to TL
msg.append({"role": "user", "content": f'{user}'}) msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create( response = openai.chat.completions.create(
temperature=0.1, temperature=0.1,
frequency_penalty=0.1, frequency_penalty=0.1,
@ -443,40 +494,47 @@ def translateText(characters, system, user, history):
) )
return response return response
def cleanTranslatedText(translatedText, varResponse): def cleanTranslatedText(translatedText, varResponse):
placeholders = { placeholders = {
f'{LANGUAGE} Translation: ': '', f"{LANGUAGE} Translation: ": "",
'Translation: ': '', "Translation: ": "",
'っ': '', "っ": "",
'〜': '~', "〜": "~",
'ッ': '', "ッ": "",
'。': '.', "。": ".",
'Placeholder Text': '' "Placeholder Text": "",
# Add more replacements as needed # Add more replacements as needed
} }
for target, replacement in placeholders.items(): for target, replacement in placeholders.items():
translatedText = translatedText.replace(target, replacement) translatedText = translatedText.replace(target, replacement)
translatedText = resubVars(translatedText, varResponse[1]) translatedText = resubVars(translatedText, varResponse[1])
if '\n' in translatedText: if "\n" in translatedText:
return [line for line in translatedText.split('\n') if line] return [line for line in translatedText.split("\n") if line]
else: else:
return [line for line in translatedText.split('\\n') if line] return [line for line in translatedText.split("\\n") if line]
def extractTranslation(translatedTextList, is_list): def extractTranslation(translatedTextList, is_list):
pattern = r'<Line(\d+)>[\\]*`?(.*?)[\\]*?`?</?Line\d+>' pattern = r"<Line(\d+)>[\\]*`?(.*?)[\\]*?`?</?Line\d+>"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item. # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list: if is_list:
return [re.findall(pattern, line)[0][1] for line in translatedTextList if re.search(pattern, line)] return [
re.findall(pattern, line)[0][1]
for line in translatedTextList
if re.search(pattern, line)
]
else: else:
matchList = re.findall(pattern, translatedTextList) matchList = re.findall(pattern, translatedTextList)
return matchList[0][1] if matchList else translatedTextList return matchList[0][1] if matchList else translatedTextList
def countTokens(characters, system, user, history): def countTokens(characters, system, user, history):
inputTotalTokens = 0 inputTotalTokens = 0
outputTotalTokens = 0 outputTotalTokens = 0
enc = tiktoken.encoding_for_model('gpt-4') enc = tiktoken.encoding_for_model("gpt-4")
# Input # Input
if isinstance(history, list): if isinstance(history, list):
for line in history: for line in history:
@ -488,15 +546,17 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user)) inputTotalTokens += len(enc.encode(user))
# Output # Output
outputTotalTokens += round(len(enc.encode(user))*3) outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens] return [inputTotalTokens, outputTotalTokens]
def combineList(tlist, text): def combineList(tlist, text):
if isinstance(text, list): if isinstance(text, list):
return [t for sublist in tlist for t in sublist] return [t for sublist in tlist for t in sublist]
return tlist[0] return tlist[0]
@retry(exceptions=Exception, tries=5, delay=5) @retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag): def translateGPT(text, history, fullPromptFlag):
totalTokens = [0, 0] totalTokens = [0, 0]
@ -508,8 +568,10 @@ def translateGPT(text, history, fullPromptFlag):
for index, tItem in enumerate(tList): for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting # Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list): if isinstance(tItem, list):
payload = '\n'.join([f'<Line{i}>`{item}`</Line{i}>' for i, item in enumerate(tItem)]) payload = "\n".join(
payload = payload.replace('``', '`Placeholder Text`') [f"<Line{i}>`{item}`</Line{i}>" for i, item in enumerate(tItem)]
)
payload = payload.replace("``", "`Placeholder Text`")
varResponse = subVars(payload) varResponse = subVars(payload)
subbedT = varResponse[0] subbedT = varResponse[0]
else: else:
@ -517,7 +579,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0] subbedT = varResponse[0]
# Things to Check before starting translation # Things to Check before starting translation
if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT): if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue continue
# Create Message # Create Message
@ -542,11 +604,13 @@ def translateGPT(text, history, fullPromptFlag):
extractedTranslations = extractTranslation(translatedTextList, True) extractedTranslations = extractTranslation(translatedTextList, True)
tList[index] = extractedTranslations tList[index] = extractedTranslations
if len(tItem) != len(translatedTextList): if len(tItem) != len(translatedTextList):
mismatch = True # Just here so breakpoint can be set mismatch = True # Just here so breakpoint can be set
history = extractedTranslations[-10:] # Update history if we have a list history = extractedTranslations[-10:] # Update history if we have a list
else: else:
# Ensure we're passing a single string to extractTranslation # Ensure we're passing a single string to extractTranslation
extractedTranslations = extractTranslation('\n'.join(translatedTextList), False) extractedTranslations = extractTranslation(
"\n".join(translatedTextList), False
)
tList[index] = extractedTranslations tList[index] = extractedTranslations
finalList = combineList(tList, text) finalList = combineList(tList, text)

87
modules/main.py
View file

@ -10,13 +10,27 @@ from dotenv import load_dotenv
# upon import, in which case if they are unset the script will crash before we can output these messages. # upon import, in which case if they are unset the script will crash before we can output these messages.
envMissing = False envMissing = False
load_dotenv() load_dotenv()
for env in ['api','key','organization','model','language','timeout','fileThreads','threads','width','listWidth']: for env in [
if os.getenv(env) is None or str(os.getenv(env))[:1] == '<': "api",
tqdm.write(Fore.RED + f'Environment variable {env} is not set!') "key",
"organization",
"model",
"language",
"timeout",
"fileThreads",
"threads",
"width",
"listWidth",
]:
if os.getenv(env) is None or str(os.getenv(env))[:1] == "<":
tqdm.write(Fore.RED + f"Environment variable {env} is not set!")
envMissing = True envMissing = True
if envMissing: if envMissing:
tqdm.write(Fore.RED + 'Some of the required environment values may not be set correctly. You can set \ tqdm.write(
these values using an .env file, for an example see .env.example') Fore.RED
+ "Some of the required environment values may not be set correctly. You can set \
these values using an .env file, for an example see .env.example"
)
from modules.rpgmakermvmz import handleMVMZ from modules.rpgmakermvmz import handleMVMZ
from modules.rpgmakerace import handleACE from modules.rpgmakerace import handleACE
@ -40,7 +54,7 @@ from modules.rpgmakerplugin import handlePlugin
# For GPT4 rate limit will be hit if you have more than 1 thread. # For GPT4 rate limit will be hit if you have more than 1 thread.
# 1 Thread for each file. Controls how many files are worked on at once. # 1 Thread for each file. Controls how many files are worked on at once.
THREADS = int(os.getenv('fileThreads')) THREADS = int(os.getenv("fileThreads"))
# [Display name, file extension, handle function] # [Display name, file extension, handle function]
MODULES = [ MODULES = [
@ -66,59 +80,76 @@ MODULES = [
] ]
# Info Message # Info Message
tqdm.write(Fore.LIGHTYELLOW_EX + "WARNING: Once the translation starts do not close it unless you want to lose your \ tqdm.write(
Fore.LIGHTYELLOW_EX
+ "WARNING: Once the translation starts do not close it unless you want to lose your \
translated data. If a file fails or gets stuck, translated lines will remain translated so you don't have \ translated data. If a file fails or gets stuck, translated lines will remain translated so you don't have \
to worry about being charged twice. You can simply copy the file generated in /translations back over to \ to worry about being charged twice. You can simply copy the file generated in /translations back over to \
/files and start the script again. It will skip over any translated text." + Fore.RESET, end='\n\n') /files and start the script again. It will skip over any translated text."
+ Fore.RESET,
end="\n\n",
)
def main(): def main():
estimate = '' estimate = ""
while estimate == '': while estimate == "":
estimate = input('Select Translation or Cost Estimation:\n\n 1. Translate\n 2. Estimate\n') estimate = input(
"Select Translation or Cost Estimation:\n\n 1. Translate\n 2. Estimate\n"
)
match estimate: match estimate:
case '1': case "1":
estimate = False estimate = False
case '2': case "2":
estimate = True estimate = True
case _: case _:
estimate = '' estimate = ""
version = '' version = ""
while True: while True:
tqdm.write("Select game engine:\n") tqdm.write("Select game engine:\n")
for position, module in enumerate(MODULES): for position, module in enumerate(MODULES):
tqdm.write(f'{str(position + 1).rjust(2)}. {module[0]} (.{module[1]})') tqdm.write(f"{str(position + 1).rjust(2)}. {module[0]} (.{module[1]})")
version = input() version = input()
try: try:
version = int(version) - 1 version = int(version) - 1
except: except:
continue continue
if version in range(len(MODULES)): if version in range(len(MODULES)):
break break
totalCost = Fore.RED + 'Translation module didn\'t return the total cost. Make sure the \ totalCost = (
files to translate are in the /files folder and that you picked the right game engine.' Fore.RED
+ "Translation module didn't return the total cost. Make sure the \
files to translate are in the /files folder and that you picked the right game engine."
)
# Open File (Threads) # Open File (Threads)
with ThreadPoolExecutor(max_workers=THREADS) as executor: with ThreadPoolExecutor(max_workers=THREADS) as executor:
futures = [executor.submit(MODULES[version][2], filename, estimate) \ futures = [
for filename in os.listdir("files") if filename.endswith(MODULES[version][1]) and filename != '.gitkeep'] executor.submit(MODULES[version][2], filename, estimate)
for filename in os.listdir("files")
if filename.endswith(MODULES[version][1]) and filename != ".gitkeep"
]
for future in as_completed(futures): for future in as_completed(futures):
try: try:
totalCost = future.result() totalCost = future.result()
except Exception as e: except Exception as e:
tracebackLineNo = str(traceback.extract_tb(sys.exc_info()[2])[-1].lineno) tracebackLineNo = str(
tqdm.write(Fore.RED + str(e) + '|' + tracebackLineNo + Fore.RESET) traceback.extract_tb(sys.exc_info()[2])[-1].lineno
)
tqdm.write(Fore.RED + str(e) + "|" + tracebackLineNo + Fore.RESET)
if totalCost != 'Fail': if totalCost != "Fail":
if estimate is False: if estimate is False:
# This is to encourage people to grab what's in /translated instead # This is to encourage people to grab what's in /translated instead
deleteFolderFiles('files') deleteFolderFiles("files")
tqdm.write(str(totalCost)) tqdm.write(str(totalCost))
def deleteFolderFiles(folderPath): def deleteFolderFiles(folderPath):
for filename in os.listdir(folderPath): for filename in os.listdir(folderPath):
file_path = os.path.join(folderPath, filename) file_path = os.path.join(folderPath, filename)
if file_path.endswith(('.json', '.yaml', '.ks')): if file_path.endswith((".json", ".yaml", ".ks")):
os.remove(file_path) os.remove(file_path)

371
modules/nscript.py
View file

@ -16,57 +16,58 @@ from tqdm import tqdm
# Open AI # Open AI
load_dotenv() load_dotenv()
if os.getenv('api').replace(' ', '') != '': if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv('api') openai.base_url = os.getenv("api")
openai.organization = os.getenv('org') openai.organization = os.getenv("org")
openai.api_key = os.getenv('key') openai.api_key = os.getenv("key")
#Globals # Globals
MODEL = os.getenv('model') MODEL = os.getenv("model")
TIMEOUT = int(os.getenv('timeout')) TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv('language').capitalize() LANGUAGE = os.getenv("language").capitalize()
PROMPT = Path('prompt.txt').read_text(encoding='utf-8') PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path('vocab.txt').read_text(encoding='utf-8') VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv('threads')) THREADS = int(os.getenv("threads"))
LOCK = threading.Lock() LOCK = threading.Lock()
WIDTH = int(os.getenv('width')) WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv('listWidth')) LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70 NOTEWIDTH = 70
MAXHISTORY = 10 MAXHISTORY = 10
ESTIMATE = '' ESTIMATE = ""
TOKENS = [0, 0] TOKENS = [0, 0]
NAMESLIST = [] NAMESLIST = []
NAMES = False # Output a list of all the character names found NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = True # Overwrites textwrap FIXTEXTWRAP = True # Overwrites textwrap
IGNORETLTEXT = False # Ignores all translated text. IGNORETLTEXT = False # Ignores all translated text.
MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
#tqdm Globals # tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0 POSITION = 0
LEAVE = False LEAVE = False
PBAR = None PBAR = None
FILENAME = None FILENAME = None
# Full Width # Full Width
ascii_to_wide = dict((i, chr(i + 0xfee0)) for i in range(0x21, 0x7f)) ascii_to_wide = dict((i, chr(i + 0xFEE0)) for i in range(0x21, 0x7F))
ascii_to_wide.update({0x20: u'\u3000', 0x2D: u'\u2212'}) # space and minus ascii_to_wide.update({0x20: "\u3000", 0x2D: "\u2212"}) # space and minus
wide_to_ascii = dict((i, chr(i - 0xfee0)) for i in range(0xff01, 0xff5f)) wide_to_ascii = dict((i, chr(i - 0xFEE0)) for i in range(0xFF01, 0xFF5F))
wide_to_ascii.update({0x3000: u' ', 0x2212: u'-'}) # space and minus wide_to_ascii.update({0x3000: " ", 0x2212: "-"}) # space and minus
# Pricing - Depends on the model https://openai.com/pricing # Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request # Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size. # If you are getting a MISMATCH LENGTH error, lower the batch size.
if 'gpt-3.5' in MODEL: if "gpt-3.5" in MODEL:
INPUTAPICOST = .002 INPUTAPICOST = 0.002
OUTPUTAPICOST = .002 OUTPUTAPICOST = 0.002
BATCHSIZE = 10 BATCHSIZE = 10
elif 'gpt-4' in MODEL: elif "gpt-4" in MODEL:
INPUTAPICOST = .005 INPUTAPICOST = 0.005
OUTPUTAPICOST = .015 OUTPUTAPICOST = 0.015
BATCHSIZE = 40 BATCHSIZE = 40
def handleOnscripter(filename, estimate): def handleOnscripter(filename, estimate):
global ESTIMATE, FILENAME global ESTIMATE, FILENAME
ESTIMATE = estimate ESTIMATE = estimate
@ -84,17 +85,21 @@ def handleOnscripter(filename, estimate):
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
# Print Total # Print Total
totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL') totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps # Print any errors on maps
if len(MISMATCH) > 0: if len(MISMATCH) > 0:
return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET return (
totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
)
else: else:
return totalString return totalString
else: else:
try: try:
with open('translated/' + filename, 'w', encoding='cp932', errors='ignore') as outFile: with open(
"translated/" + filename, "w", encoding="cp932", errors="ignore"
) as outFile:
start = time.time() start = time.time()
translatedData = openFiles(filename) translatedData = openFiles(filename)
@ -107,23 +112,36 @@ def handleOnscripter(filename, estimate):
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
except Exception: except Exception:
traceback.print_exc() traceback.print_exc()
return 'Fail' return "Fail"
return getResultString(["", TOKENS, None], end - start, "TOTAL")
return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def getResultString(translatedData, translationTime, filename): def getResultString(translatedData, translationTime, filename):
# File Print String # File Print String
totalTokenstring =\ totalTokenstring = (
Fore.YELLOW +\ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
'[Input: ' + str(translatedData[1][0]) + ']'\ "[Output: "
'[Output: ' + str(translatedData[1][1]) + ']'\ + str(translatedData[1][1])
'[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\ + "]" "[Cost: ${:,.4f}".format(
(translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']' (translatedData[1][0] * 0.001 * INPUTAPICOST)
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
)
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None: if translatedData[2] == None:
# Success # Success
return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET return (
filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.GREEN
+ " \u2713 "
+ Fore.RESET
)
else: else:
# Fail # Fail
@ -132,31 +150,41 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
errorString = str(e) + Fore.RED errorString = str(e) + Fore.RED
return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\ return (
errorString + Fore.RESET filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.RED
+ " \u2717 "
+ errorString
+ Fore.RESET
)
def openFiles(filename): def openFiles(filename):
with open('files/' + filename, 'r', encoding='cp932') as readFile: with open("files/" + filename, "r", encoding="cp932") as readFile:
translatedData = parseOnscripter(readFile, filename) translatedData = parseOnscripter(readFile, filename)
# Delete lines marked for deletion # Delete lines marked for deletion
finalData = [] finalData = []
for line in translatedData[0]: for line in translatedData[0]:
if line != '\\d\n': if line != "\\d\n":
finalData.append(line) finalData.append(line)
translatedData[0] = finalData translatedData[0] = finalData
return translatedData return translatedData
def parseOnscripter(readFile, filename): def parseOnscripter(readFile, filename):
totalTokens = [0,0] totalTokens = [0, 0]
# Read File into data # Read File into data
data = readFile.readlines() data = readFile.readlines()
# Create Progress Bar # Create Progress Bar
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
pbar.desc=filename pbar.desc = filename
try: try:
result = translateOnscripter(data, pbar, filename, []) result = translateOnscripter(data, pbar, filename, [])
@ -167,11 +195,12 @@ def parseOnscripter(readFile, filename):
return [data, totalTokens, e] return [data, totalTokens, e]
return [data, totalTokens, None] return [data, totalTokens, None]
def translateOnscripter(data, pbar, filename, translatedList): def translateOnscripter(data, pbar, filename, translatedList):
stringList = [] stringList = []
currentGroup = [] currentGroup = []
tokens = [0,0] tokens = [0, 0]
speaker = '' speaker = ""
voice = False voice = False
global LOCK, ESTIMATE, PBAR global LOCK, ESTIMATE, PBAR
PBAR = pbar PBAR = pbar
@ -180,38 +209,38 @@ def translateOnscripter(data, pbar, filename, translatedList):
# Dialogue # Dialogue
while i < len(data): while i < len(data):
# Lines # Lines
regex = r'^([\u3000「（【[][^\n]+)' regex = r"^([\u3000「（【[][^\n]+)"
match = re.search(regex, data[i]) match = re.search(regex, data[i])
if match != None and match.group(1) != '': if match != None and match.group(1) != "":
originalString = match.group(1) originalString = match.group(1)
# Pass 1 # Pass 1
if translatedList == []: if translatedList == []:
# Grab Consecutive Strings # Grab Consecutive Strings
jaString = match.group(1) jaString = match.group(1)
while len(data) > i+1 and re.match(regex, data[i+1]): while len(data) > i + 1 and re.match(regex, data[i + 1]):
data[i] = '' data[i] = ""
i += 1 i += 1
jaString = f'{jaString} {data[i]}' jaString = f"{jaString} {data[i]}"
# Convert from Wide # Convert from Wide
jaString = jaString.translate(wide_to_ascii) jaString = jaString.translate(wide_to_ascii)
# Remove any textwrap and \u3000 and \ # Remove any textwrap and \u3000 and \
jaString = jaString.replace('\n', '') jaString = jaString.replace("\n", "")
jaString = jaString.replace('\u3000', '') jaString = jaString.replace("\u3000", "")
jaString = jaString.replace('\\', '') jaString = jaString.replace("\\", "")
jaString = jaString.replace('　＞', ')') jaString = jaString.replace("　＞", ")")
jaString = jaString.replace('＜　', '(') jaString = jaString.replace("＜　", "(")
# Remove Furigana # Remove Furigana
furiMatch = re.findall(r'({(.+?)\/(.+?)})', jaString) furiMatch = re.findall(r"({(.+?)\/(.+?)})", jaString)
if furiMatch: if furiMatch:
for match in furiMatch: for match in furiMatch:
jaString = jaString.replace(match[0], match[2]) jaString = jaString.replace(match[0], match[2])
# Add String # Add String
stringList.append(jaString.strip()) stringList.append(jaString.strip())
# Pass 2 # Pass 2
else: else:
# Get Text # Get Text
@ -226,24 +255,24 @@ def translateOnscripter(data, pbar, filename, translatedList):
# Textwrap & Other Text # Textwrap & Other Text
translatedText = textwrap.fill(translatedText, width=WIDTH) translatedText = textwrap.fill(translatedText, width=WIDTH)
translatedText = translatedText.replace('\n', '\n\u3000') translatedText = translatedText.replace("\n", "\n\u3000")
# Split the string into lines # Split the string into lines
lines = translatedText.split('\n') lines = translatedText.split("\n")
# Add a backslash after every 3rd line # Add a backslash after every 3rd line
j = 0 j = 0
while j < len(lines): while j < len(lines):
if j == 4: if j == 4:
lines[j-1] = f'{lines[j-1]}\\' lines[j - 1] = f"{lines[j-1]}\\"
lines[j] = f'\n{lines[j]}' lines[j] = f"\n{lines[j]}"
j += 1 j += 1
# Join the lines back into a single string # Join the lines back into a single string
translatedText = '\n'.join(lines) translatedText = "\n".join(lines)
# Remove Double Spaces # Remove Double Spaces
translatedText = translatedText.replace(' ', ' ') translatedText = translatedText.replace(" ", " ")
# Convert to Wide # Convert to Wide
translatedText = translatedText.translate(ascii_to_wide) translatedText = translatedText.translate(ascii_to_wide)
@ -252,18 +281,20 @@ def translateOnscripter(data, pbar, filename, translatedList):
translatedText = fixText(translatedText) translatedText = fixText(translatedText)
# Set Data # Set Data
data[i] = data[i].replace(originalString, f'{translatedText}') data[i] = data[i].replace(originalString, f"{translatedText}")
i += 1 i += 1
# Choices # Choices
elif 'csel' in data[i] and translatedList != []: elif "csel" in data[i] and translatedList != []:
choiceList = [] choiceList = []
jaString = data[i] jaString = data[i]
choiceList = re.findall(r'\"(.*?)\"', jaString) choiceList = re.findall(r"\"(.*?)\"", jaString)
if len(choiceList) > 0: if len(choiceList) > 0:
# Translate # Translate
response = translateGPT(choiceList, 'This will be a dialogue option', True) response = translateGPT(
choiceList, "This will be a dialogue option", True
)
translatedTextList = response[0] translatedTextList = response[0]
tokens[0] += response[1][0] tokens[0] += response[1][0]
tokens[1] += response[1][1] tokens[1] += response[1][1]
@ -286,9 +317,9 @@ def translateOnscripter(data, pbar, filename, translatedList):
# Set Progress # Set Progress
pbar.total = len(stringList) pbar.total = len(stringList)
pbar.refresh() pbar.refresh()
# Translate # Translate
response = translateGPT(stringList, '', True) response = translateGPT(stringList, "", True)
tokens[0] += response[1][0] tokens[0] += response[1][0]
tokens[1] += response[1][1] tokens[1] += response[1][1]
translatedList = response[0] translatedList = response[0]
@ -304,54 +335,72 @@ def translateOnscripter(data, pbar, filename, translatedList):
MISMATCH.append(filename) MISMATCH.append(filename)
return tokens return tokens
def fixText(translatedText): def fixText(translatedText):
# Add Break # Add Break
translatedText = translatedText.replace('\"', '\'') translatedText = translatedText.replace('"', "'")
translatedText = f'\u3000{translatedText}\\' translatedText = f"\u3000{translatedText}\\"
# Unconvert Codes # Unconvert Codes
matchList = re.findall(r'([＄].+?)[^\w]', translatedText) matchList = re.findall(r"([＄].+?)[^\w]", translatedText)
if matchList: if matchList:
for match in matchList: for match in matchList:
translatedText = translatedText.replace(match, match.translate(wide_to_ascii)) translatedText = translatedText.replace(
match, match.translate(wide_to_ascii)
)
# Unconvert Color Codes # Unconvert Color Codes
matchList = re.findall(r'([＃][\w\d]{6})', translatedText) matchList = re.findall(r"([＃][\w\d]{6})", translatedText)
if matchList: if matchList:
for match in matchList: for match in matchList:
translatedText = translatedText.replace(match, match.translate(wide_to_ascii)) translatedText = translatedText.replace(
match, match.translate(wide_to_ascii)
)
# Unconvert Variables # Unconvert Variables
matchList = re.findall(r'([％]\w.+?)[^\w＿]', translatedText) matchList = re.findall(r"([％]\w.+?)[^\w＿]", translatedText)
if matchList: if matchList:
for match in matchList: for match in matchList:
translatedText = translatedText.replace(match, match.translate(wide_to_ascii)) translatedText = translatedText.replace(
match, match.translate(wide_to_ascii)
)
# Unconvert Backslashes # Unconvert Backslashes
matchList = re.findall(r'＼', translatedText) matchList = re.findall(r"＼", translatedText)
if matchList: if matchList:
for match in matchList: for match in matchList:
translatedText = translatedText.replace(match, match.translate(wide_to_ascii)) translatedText = translatedText.replace(
match, match.translate(wide_to_ascii)
)
return translatedText return translatedText
# Save some money and enter the character before translation # Save some money and enter the character before translation
def getSpeaker(speaker): def getSpeaker(speaker):
match speaker: match speaker:
case 'ファイン': case "ファイン":
return ['Fine', [0,0]] return ["Fine", [0, 0]]
case '': case "":
return ['', [0,0]] return ["", [0, 0]]
case _: case _:
# Store Speaker # Store Speaker
if speaker not in str(NAMESLIST): if speaker not in str(NAMESLIST):
response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False) response = translateGPT(
speaker,
"Reply with the " + LANGUAGE + " translation of the NPC name.",
False,
)
response[0] = response[0].title() response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s") response[0] = response[0].replace("'S", "'s")
# Retry if name doesn't translate for some reason # Retry if name doesn't translate for some reason
if re.search(r'([a-zA-Z？?])', response[0]) == None: if re.search(r"([a-zA-Z？?])", response[0]) == None:
response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False) response = translateGPT(
speaker,
"Reply with the " + LANGUAGE + " translation of the NPC name.",
False,
)
response[0] = response[0].title() response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s") response[0] = response[0].replace("'S", "'s")
@ -362,74 +411,76 @@ def getSpeaker(speaker):
else: else:
for i in range(len(NAMESLIST)): for i in range(len(NAMESLIST)):
if speaker == NAMESLIST[i][0]: if speaker == NAMESLIST[i][0]:
return [NAMESLIST[i][1],[0,0]] return [NAMESLIST[i][1], [0, 0]]
return [speaker,[0,0]] return [speaker, [0, 0]]
def subVars(jaString): def subVars(jaString):
jaString = jaString.replace('\u3000', ' ') jaString = jaString.replace("\u3000", " ")
# Nested # Nested
count = 0 count = 0
nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString) nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList) nestedList = set(nestedList)
if len(nestedList) != 0: if len(nestedList) != 0:
for icon in nestedList: for icon in nestedList:
jaString = jaString.replace(icon, '[Nested_' + str(count) + ']') jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString) iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList) iconList = set(iconList)
if len(iconList) != 0: if len(iconList) != 0:
for icon in iconList: for icon in iconList:
jaString = jaString.replace(icon, '[Ascii_' + str(count) + ']') jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
colorList = re.findall(r'([\\]+c\[\d+\][\\]+c|[\\]+c\[\d+\])', jaString) colorList = re.findall(r"([\\]+c\[\d+\][\\]+c|[\\]+c\[\d+\])", jaString)
colorList = set(colorList) colorList = set(colorList)
if len(colorList) != 0: if len(colorList) != 0:
for color in colorList: for color in colorList:
jaString = jaString.replace(color, '[Color_' + str(count) + ']') jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1 count += 1
# Names # Names
count = 0 count = 0
nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString) nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList) nameList = set(nameList)
if len(nameList) != 0: if len(nameList) != 0:
for name in nameList: for name in nameList:
jaString = jaString.replace(name, '[Noun_' + str(count) + ']') jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1 count += 1
# Variables # Variables
count = 0 count = 0
varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString) varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList) varList = set(varList)
if len(varList) != 0: if len(varList) != 0:
for var in varList: for var in varList:
jaString = jaString.replace(var, '[Var_' + str(count) + ']') jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
formatList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString) formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList) formatList = set(formatList)
if len(formatList) != 0: if len(formatList) != 0:
for var in formatList: for var in formatList:
jaString = jaString.replace(var, '[FCode_' + str(count) + ']') jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1 count += 1
# Put all lists in list and return # Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList] allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList] return [jaString, allList]
def resubVars(translatedText, allList): def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense # Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r'\[\s?.+?\s?\]', translatedText) matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0: if len(matchList) > 0:
for match in matchList: for match in matchList:
text = match.strip() text = match.strip()
@ -439,54 +490,58 @@ def resubVars(translatedText, allList):
count = 0 count = 0
if len(allList[0]) != 0: if len(allList[0]) != 0:
for var in allList[0]: for var in allList[0]:
translatedText = translatedText.replace('[Nested_' + str(count) + ']', var) translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
if len(allList[1]) != 0: if len(allList[1]) != 0:
for var in allList[1]: for var in allList[1]:
translatedText = translatedText.replace('[Ascii_' + str(count) + ']', var) translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
if len(allList[2]) != 0: if len(allList[2]) != 0:
for var in allList[2]: for var in allList[2]:
translatedText = translatedText.replace('[Color_' + str(count) + ']', var) translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1 count += 1
# Names # Names
count = 0 count = 0
if len(allList[3]) != 0: if len(allList[3]) != 0:
for var in allList[3]: for var in allList[3]:
translatedText = translatedText.replace('[Noun_' + str(count) + ']', var) translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1 count += 1
# Vars # Vars
count = 0 count = 0
if len(allList[4]) != 0: if len(allList[4]) != 0:
for var in allList[4]: for var in allList[4]:
translatedText = translatedText.replace('[Var_' + str(count) + ']', var) translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
if len(allList[5]) != 0: if len(allList[5]) != 0:
for var in allList[5]: for var in allList[5]:
translatedText = translatedText.replace('[FCode_' + str(count) + ']', var) translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1 count += 1
return translatedText return translatedText
def batchList(input_list, batch_size): def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0: if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer") raise ValueError("batch_size must be a positive integer")
return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)] return [
input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
]
def createContext(fullPromptFlag, subbedT): def createContext(fullPromptFlag, subbedT):
characters = 'Game Characters:\n\ characters = "Game Characters:\n\
レナリス (Renalith) - Female\n\ レナリス (Renalith) - Female\n\
スクルー (Sukuru) - Female\n\ スクルー (Sukuru) - Female\n\
シスターミサ (Sister Misa) - Female\n\ シスターミサ (Sister Misa) - Female\n\
@ -514,10 +569,12 @@ def createContext(fullPromptFlag, subbedT):
エメルーラ (Emerald) - Female\n\ エメルーラ (Emerald) - Female\n\
フンシス (Funsis) - Male \n\ フンシス (Funsis) - Male \n\
バゼット (Bazzet) - Female\n\ バゼット (Bazzet) - Female\n\
' "
system = PROMPT + VOCAB if fullPromptFlag else \ system = (
f"\ PROMPT + VOCAB
if fullPromptFlag
else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\ - You translate everything, including erotic and explicit content.\n\
@ -529,9 +586,11 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\ {VOCAB}\n\
" "
user = f'```json\n{subbedT}```' )
user = f"```json\n{subbedT}```"
return characters, system, user return characters, system, user
def translateText(characters, system, user, history, penalty): def translateText(characters, system, user, history, penalty):
# Prompt # Prompt
msg = [{"role": "system", "content": system + characters}] msg = [{"role": "system", "content": system + characters}]
@ -544,30 +603,31 @@ def translateText(characters, system, user, history, penalty):
msg.extend([{"role": "system", "content": h} for h in history]) msg.extend([{"role": "system", "content": h} for h in history])
else: else:
msg.append({"role": "system", "content": history}) msg.append({"role": "system", "content": history})
# Content to TL # Content to TL
msg.append({"role": "user", "content": f'{user}'}) msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create( response = openai.chat.completions.create(
temperature=0, temperature=0,
frequency_penalty=penalty, frequency_penalty=penalty,
model=MODEL, model=MODEL,
response_format={ "type": "json_object" }, response_format={"type": "json_object"},
messages=msg, messages=msg,
) )
return response return response
def cleanTranslatedText(translatedText, varResponse): def cleanTranslatedText(translatedText, varResponse):
placeholders = { placeholders = {
f'{LANGUAGE} Translation: ': '', f"{LANGUAGE} Translation: ": "",
'Translation: ': '', "Translation: ": "",
'っ': '', "っ": "",
'〜': '~', "〜": "~",
'ッ': '', "ッ": "",
'。': '.', "。": ".",
'「': '\\"', "「": '\\"',
'」': '\\"', "」": '\\"',
'- ': '-', "- ": "-",
'Placeholder Text': '', "Placeholder Text": "",
# Add more replacements as needed # Add more replacements as needed
} }
for target, replacement in placeholders.items(): for target, replacement in placeholders.items():
@ -578,11 +638,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1]) translatedText = resubVars(translatedText, varResponse[1])
return translatedText return translatedText
def elongateCharacters(text): def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters # Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character # Using a positive lookbehind assertion to capture the preceding character
pattern = r'(?<=(.))ー+' pattern = r"(?<=(.))ー+"
# Define a replacement function that elongates the captured character # Define a replacement function that elongates the captured character
def repl(match): def repl(match):
char = match.group(1) # The character before the ー sequence char = match.group(1) # The character before the ー sequence
@ -592,6 +653,7 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text # Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text) return re.sub(pattern, repl, text)
def extractTranslation(translatedTextList, is_list): def extractTranslation(translatedTextList, is_list):
try: try:
line_dict = json.loads(translatedTextList) line_dict = json.loads(translatedTextList)
@ -603,11 +665,12 @@ def extractTranslation(translatedTextList, is_list):
print(e) print(e)
return translatedTextList return translatedTextList
def countTokens(characters, system, user, history): def countTokens(characters, system, user, history):
inputTotalTokens = 0 inputTotalTokens = 0
outputTotalTokens = 0 outputTotalTokens = 0
enc = tiktoken.encoding_for_model('gpt-4') enc = tiktoken.encoding_for_model("gpt-4")
# Input # Input
if isinstance(history, list): if isinstance(history, list):
for line in history: for line in history:
@ -619,19 +682,21 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user)) inputTotalTokens += len(enc.encode(user))
# Output # Output
outputTotalTokens += round(len(enc.encode(user))*3) outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens] return [inputTotalTokens, outputTotalTokens]
def combineList(tlist, text): def combineList(tlist, text):
if isinstance(text, list): if isinstance(text, list):
return [t for sublist in tlist for t in sublist] return [t for sublist in tlist for t in sublist]
return tlist[0] return tlist[0]
@retry(exceptions=Exception, tries=5, delay=5) @retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag): def translateGPT(text, history, fullPromptFlag):
global PBAR global PBAR
mismatch = False mismatch = False
totalTokens = [0, 0] totalTokens = [0, 0]
if isinstance(text, list): if isinstance(text, list):
@ -651,7 +716,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0] subbedT = varResponse[0]
# Things to Check before starting translation # Things to Check before starting translation
if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT): if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
if PBAR is not None: if PBAR is not None:
PBAR.update(len(tItem)) PBAR.update(len(tItem))
continue continue
@ -688,12 +753,14 @@ def translateGPT(text, history, fullPromptFlag):
if isinstance(tItem, list): if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True) extractedTranslations = extractTranslation(translatedText, True)
if len(tItem) != len(extractedTranslations): if len(tItem) != len(extractedTranslations):
mismatch = True # Just here for breakpoint mismatch = True # Just here for breakpoint
# Set if no mismatch # Set if no mismatch
if mismatch == False: if mismatch == False:
tList[index] = extractedTranslations tList[index] = extractedTranslations
history = extractedTranslations[-10:] # Update history if we have a list history = extractedTranslations[
-10:
] # Update history if we have a list
else: else:
history = text[-10:] history = text[-10:]
mismatch = False mismatch = False

292
modules/regex.py
View file

@ -15,49 +15,50 @@ from tqdm import tqdm
# Open AI # Open AI
load_dotenv() load_dotenv()
if os.getenv('api').replace(' ', '') != '': if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv('api') openai.base_url = os.getenv("api")
openai.organization = os.getenv('org') openai.organization = os.getenv("org")
openai.api_key = os.getenv('key') openai.api_key = os.getenv("key")
#Globals # Globals
MODEL = os.getenv('model') MODEL = os.getenv("model")
TIMEOUT = int(os.getenv('timeout')) TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv('language').capitalize() LANGUAGE = os.getenv("language").capitalize()
PROMPT = Path('prompt.txt').read_text(encoding='utf-8') PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path('vocab.txt').read_text(encoding='utf-8') VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv('threads')) THREADS = int(os.getenv("threads"))
LOCK = threading.Lock() LOCK = threading.Lock()
WIDTH = int(os.getenv('width')) WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv('listWidth')) LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70 NOTEWIDTH = 70
MAXHISTORY = 10 MAXHISTORY = 10
ESTIMATE = '' ESTIMATE = ""
TOKENS = [0, 0] TOKENS = [0, 0]
NAMESLIST = [] NAMESLIST = []
NAMES = False # Output a list of all the character names found NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = True # Overwrites textwrap FIXTEXTWRAP = True # Overwrites textwrap
IGNORETLTEXT = False # Ignores all translated text. IGNORETLTEXT = False # Ignores all translated text.
MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
#tqdm Globals # tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0 POSITION = 0
LEAVE = False LEAVE = False
# Pricing - Depends on the model https://openai.com/pricing # Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request # Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size. # If you are getting a MISMATCH LENGTH error, lower the batch size.
if 'gpt-3.5' in MODEL: if "gpt-3.5" in MODEL:
INPUTAPICOST = .002 INPUTAPICOST = 0.002
OUTPUTAPICOST = .002 OUTPUTAPICOST = 0.002
BATCHSIZE = 10 BATCHSIZE = 10
elif 'gpt-4' in MODEL: elif "gpt-4" in MODEL:
INPUTAPICOST = .005 INPUTAPICOST = 0.005
OUTPUTAPICOST = .015 OUTPUTAPICOST = 0.015
BATCHSIZE = 40 BATCHSIZE = 40
def handleRegex(filename, estimate): def handleRegex(filename, estimate):
global ESTIMATE global ESTIMATE
ESTIMATE = estimate ESTIMATE = estimate
@ -74,17 +75,21 @@ def handleRegex(filename, estimate):
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
# Print Total # Print Total
totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL') totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps # Print any errors on maps
if len(MISMATCH) > 0: if len(MISMATCH) > 0:
return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET return (
totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
)
else: else:
return totalString return totalString
else: else:
try: try:
with open('translated/' + filename, 'w', encoding='cp932', errors='ignore') as outFile: with open(
"translated/" + filename, "w", encoding="cp932", errors="ignore"
) as outFile:
start = time.time() start = time.time()
translatedData = openFiles(filename) translatedData = openFiles(filename)
@ -97,23 +102,36 @@ def handleRegex(filename, estimate):
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
except Exception: except Exception:
traceback.print_exc() traceback.print_exc()
return 'Fail' return "Fail"
return getResultString(["", TOKENS, None], end - start, "TOTAL")
return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def getResultString(translatedData, translationTime, filename): def getResultString(translatedData, translationTime, filename):
# File Print String # File Print String
totalTokenstring =\ totalTokenstring = (
Fore.YELLOW +\ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
'[Input: ' + str(translatedData[1][0]) + ']'\ "[Output: "
'[Output: ' + str(translatedData[1][1]) + ']'\ + str(translatedData[1][1])
'[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\ + "]" "[Cost: ${:,.4f}".format(
(translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']' (translatedData[1][0] * 0.001 * INPUTAPICOST)
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
)
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None: if translatedData[2] == None:
# Success # Success
return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET return (
filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.GREEN
+ " \u2713 "
+ Fore.RESET
)
else: else:
# Fail # Fail
@ -122,31 +140,41 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
errorString = str(e) + Fore.RED errorString = str(e) + Fore.RED
return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\ return (
errorString + Fore.RESET filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.RED
+ " \u2717 "
+ errorString
+ Fore.RESET
)
def openFiles(filename): def openFiles(filename):
with open('files/' + filename, 'r', encoding='shift_jis') as readFile: with open("files/" + filename, "r", encoding="shift_jis") as readFile:
translatedData = parseRegex(readFile, filename) translatedData = parseRegex(readFile, filename)
# Delete lines marked for deletion # Delete lines marked for deletion
finalData = [] finalData = []
for line in translatedData[0]: for line in translatedData[0]:
if line != '\\d\n': if line != "\\d\n":
finalData.append(line) finalData.append(line)
translatedData[0] = finalData translatedData[0] = finalData
return translatedData return translatedData
def parseRegex(readFile, filename): def parseRegex(readFile, filename):
totalTokens = [0,0] totalTokens = [0, 0]
# Read File into data # Read File into data
data = readFile.readlines() data = readFile.readlines()
# Create Progress Bar # Create Progress Bar
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
pbar.desc=filename pbar.desc = filename
try: try:
result = translateRegex(data, pbar, filename, []) result = translateRegex(data, pbar, filename, [])
@ -157,36 +185,37 @@ def parseRegex(readFile, filename):
return [data, totalTokens, e] return [data, totalTokens, e]
return [data, totalTokens, None] return [data, totalTokens, None]
def translateRegex(data, pbar, filename, translatedList): def translateRegex(data, pbar, filename, translatedList):
stringList = [] stringList = []
currentGroup = [] currentGroup = []
tokens = [0,0] tokens = [0, 0]
speaker = '' speaker = ""
voice = False voice = False
global LOCK, ESTIMATE global LOCK, ESTIMATE
i = 0 i = 0
while i < len(data): while i < len(data):
voice = False voice = False
speaker = '' speaker = ""
if 'actorID' in data[i]: if "actorID" in data[i]:
# Lines # Lines
match = re.search(r'label[\\]+\":[\\]+\"(.*?)\"', data[i]) match = re.search(r"label[\\]+\":[\\]+\"(.*?)\"", data[i])
if match == None: if match == None:
match = re.search(r'label[\\]+\":[\\]+\"(.*?)\"', data[i]) match = re.search(r"label[\\]+\":[\\]+\"(.*?)\"", data[i])
if match != None and match.group(1) != '': if match != None and match.group(1) != "":
originalString = match.group(1) originalString = match.group(1)
# Pass 1 # Pass 1
if translatedList == []: if translatedList == []:
# Grab Consecutive Strings # Grab Consecutive Strings
jaString = match.group(1) jaString = match.group(1)
# Remove any textwrap # Remove any textwrap
jaString = jaString.replace('\n', ' ') jaString = jaString.replace("\n", " ")
# Add String # Add String
stringList.append(jaString.strip()) stringList.append(jaString.strip())
# Pass 2 # Pass 2
else: else:
# Get Text # Get Text
@ -217,9 +246,15 @@ def translateRegex(data, pbar, filename, translatedList):
# Set Progress # Set Progress
pbar.total = len(stringList) pbar.total = len(stringList)
pbar.refresh() pbar.refresh()
# Translate # Translate
response = translateGPT(stringList, 'Reply with the English TL of the NPC Name', True, pbar, filename) response = translateGPT(
stringList,
"Reply with the English TL of the NPC Name",
True,
pbar,
filename,
)
tokens[0] += response[1][0] tokens[0] += response[1][0]
tokens[1] += response[1][1] tokens[1] += response[1][1]
translatedList = response[0] translatedList = response[0]
@ -235,94 +270,103 @@ def translateRegex(data, pbar, filename, translatedList):
MISMATCH.append(filename) MISMATCH.append(filename)
return tokens return tokens
# Save some money and enter the character before translation # Save some money and enter the character before translation
def getSpeaker(speaker, pbar, filename): def getSpeaker(speaker, pbar, filename):
match speaker: match speaker:
case 'ファイン': case "ファイン":
return ['Fine', [0,0]] return ["Fine", [0, 0]]
case '': case "":
return ['', [0,0]] return ["", [0, 0]]
case _: case _:
# Store Speaker # Store Speaker
if speaker not in str(NAMESLIST): if speaker not in str(NAMESLIST):
response = translateGPT(speaker, 'Reply with only the '+ LANGUAGE +' translation of the NPC name.', False, pbar, filename) response = translateGPT(
speaker,
"Reply with only the " + LANGUAGE + " translation of the NPC name.",
False,
pbar,
filename,
)
response[0] = response[0].replace("'S", "'s") response[0] = response[0].replace("'S", "'s")
speakerList = [speaker, response[0]] speakerList = [speaker, response[0]]
NAMESLIST.append(speakerList) NAMESLIST.append(speakerList)
return response return response
# Find Speaker # Find Speaker
else: else:
for i in range(len(NAMESLIST)): for i in range(len(NAMESLIST)):
if speaker == NAMESLIST[i][0]: if speaker == NAMESLIST[i][0]:
return [NAMESLIST[i][1],[0,0]] return [NAMESLIST[i][1], [0, 0]]
return [speaker,[0,0]] return [speaker, [0, 0]]
def subVars(jaString): def subVars(jaString):
jaString = jaString.replace('\u3000', ' ') jaString = jaString.replace("\u3000", " ")
# Nested # Nested
count = 0 count = 0
nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString) nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList) nestedList = set(nestedList)
if len(nestedList) != 0: if len(nestedList) != 0:
for icon in nestedList: for icon in nestedList:
jaString = jaString.replace(icon, '[Nested_' + str(count) + ']') jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString) iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList) iconList = set(iconList)
if len(iconList) != 0: if len(iconList) != 0:
for icon in iconList: for icon in iconList:
jaString = jaString.replace(icon, '[Ascii_' + str(count) + ']') jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString) colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList) colorList = set(colorList)
if len(colorList) != 0: if len(colorList) != 0:
for color in colorList: for color in colorList:
jaString = jaString.replace(color, '[Color_' + str(count) + ']') jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1 count += 1
# Names # Names
count = 0 count = 0
nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString) nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList) nameList = set(nameList)
if len(nameList) != 0: if len(nameList) != 0:
for name in nameList: for name in nameList:
jaString = jaString.replace(name, '[Noun_' + str(count) + ']') jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1 count += 1
# Variables # Variables
count = 0 count = 0
varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString) varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList) varList = set(varList)
if len(varList) != 0: if len(varList) != 0:
for var in varList: for var in varList:
jaString = jaString.replace(var, '[Var_' + str(count) + ']') jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
formatList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString) formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList) formatList = set(formatList)
if len(formatList) != 0: if len(formatList) != 0:
for var in formatList: for var in formatList:
jaString = jaString.replace(var, '[FCode_' + str(count) + ']') jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1 count += 1
# Put all lists in list and return # Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList] allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList] return [jaString, allList]
def resubVars(translatedText, allList): def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense # Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r'\[\s?.+?\s?\]', translatedText) matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0: if len(matchList) > 0:
for match in matchList: for match in matchList:
text = match.strip() text = match.strip()
@ -332,54 +376,58 @@ def resubVars(translatedText, allList):
count = 0 count = 0
if len(allList[0]) != 0: if len(allList[0]) != 0:
for var in allList[0]: for var in allList[0]:
translatedText = translatedText.replace('[Nested_' + str(count) + ']', var) translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
if len(allList[1]) != 0: if len(allList[1]) != 0:
for var in allList[1]: for var in allList[1]:
translatedText = translatedText.replace('[Ascii_' + str(count) + ']', var) translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
if len(allList[2]) != 0: if len(allList[2]) != 0:
for var in allList[2]: for var in allList[2]:
translatedText = translatedText.replace('[Color_' + str(count) + ']', var) translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1 count += 1
# Names # Names
count = 0 count = 0
if len(allList[3]) != 0: if len(allList[3]) != 0:
for var in allList[3]: for var in allList[3]:
translatedText = translatedText.replace('[Noun_' + str(count) + ']', var) translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1 count += 1
# Vars # Vars
count = 0 count = 0
if len(allList[4]) != 0: if len(allList[4]) != 0:
for var in allList[4]: for var in allList[4]:
translatedText = translatedText.replace('[Var_' + str(count) + ']', var) translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
if len(allList[5]) != 0: if len(allList[5]) != 0:
for var in allList[5]: for var in allList[5]:
translatedText = translatedText.replace('[FCode_' + str(count) + ']', var) translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1 count += 1
return translatedText return translatedText
def batchList(input_list, batch_size): def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0: if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer") raise ValueError("batch_size must be a positive integer")
return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)] return [
input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
]
def createContext(fullPromptFlag, subbedT): def createContext(fullPromptFlag, subbedT):
characters = 'Game Characters:\n\ characters = "Game Characters:\n\
フィリア (Philia) - Female\n\ フィリア (Philia) - Female\n\
アルネット (Annett) - Female\n\ アルネット (Annett) - Female\n\
ラピュセナ (Rapusena) - Female\n\ ラピュセナ (Rapusena) - Female\n\
@ -389,10 +437,12 @@ def createContext(fullPromptFlag, subbedT):
カルナ (Karna) - Female\n\ カルナ (Karna) - Female\n\
ラフィング＝スピア (Laughing Spear) - Female\n\ ラフィング＝スピア (Laughing Spear) - Female\n\
ノーラ (Nora) - Female\n\ ノーラ (Nora) - Female\n\
' "
system = PROMPT + VOCAB if fullPromptFlag else \ system = (
f"\ PROMPT + VOCAB
if fullPromptFlag
else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\ - You translate everything, including erotic and explicit content.\n\
@ -404,9 +454,11 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\ {VOCAB}\n\
" "
user = f'{subbedT}' )
user = f"{subbedT}"
return characters, system, user return characters, system, user
def translateText(characters, system, user, history): def translateText(characters, system, user, history):
# Prompt # Prompt
msg = [{"role": "system", "content": system + characters}] msg = [{"role": "system", "content": system + characters}]
@ -419,9 +471,9 @@ def translateText(characters, system, user, history):
msg.extend([{"role": "system", "content": h} for h in history]) msg.extend([{"role": "system", "content": h} for h in history])
else: else:
msg.append({"role": "system", "content": history}) msg.append({"role": "system", "content": history})
# Content to TL # Content to TL
msg.append({"role": "user", "content": f'{user}'}) msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create( response = openai.chat.completions.create(
temperature=0.1, temperature=0.1,
frequency_penalty=0.1, frequency_penalty=0.1,
@ -430,15 +482,16 @@ def translateText(characters, system, user, history):
) )
return response return response
def cleanTranslatedText(translatedText, varResponse): def cleanTranslatedText(translatedText, varResponse):
placeholders = { placeholders = {
f'{LANGUAGE} Translation: ': '', f"{LANGUAGE} Translation: ": "",
'Translation: ': '', "Translation: ": "",
'っ': '', "っ": "",
'〜': '~', "〜": "~",
'ッ': '', "ッ": "",
'。': '.', "。": ".",
'Placeholder Text': '' "Placeholder Text": "",
# Add more replacements as needed # Add more replacements as needed
} }
for target, replacement in placeholders.items(): for target, replacement in placeholders.items():
@ -449,11 +502,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1]) translatedText = resubVars(translatedText, varResponse[1])
return translatedText return translatedText
def elongateCharacters(text): def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters # Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character # Using a positive lookbehind assertion to capture the preceding character
pattern = r'(?<=(.))ー+' pattern = r"(?<=(.))ー+"
# Define a replacement function that elongates the captured character # Define a replacement function that elongates the captured character
def repl(match): def repl(match):
char = match.group(1) # The character before the ー sequence char = match.group(1) # The character before the ー sequence
@ -463,8 +517,9 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text # Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text) return re.sub(pattern, repl, text)
def extractTranslation(translatedTextList, is_list): def extractTranslation(translatedTextList, is_list):
pattern = r'`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?' pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item. # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list: if is_list:
matchList = re.findall(pattern, translatedTextList) matchList = re.findall(pattern, translatedTextList)
@ -473,11 +528,12 @@ def extractTranslation(translatedTextList, is_list):
matchList = re.findall(pattern, translatedTextList) matchList = re.findall(pattern, translatedTextList)
return matchList[0][0] if matchList else translatedTextList return matchList[0][0] if matchList else translatedTextList
def countTokens(characters, system, user, history): def countTokens(characters, system, user, history):
inputTotalTokens = 0 inputTotalTokens = 0
outputTotalTokens = 0 outputTotalTokens = 0
enc = tiktoken.encoding_for_model('gpt-4') enc = tiktoken.encoding_for_model("gpt-4")
# Input # Input
if isinstance(history, list): if isinstance(history, list):
for line in history: for line in history:
@ -489,15 +545,17 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user)) inputTotalTokens += len(enc.encode(user))
# Output # Output
outputTotalTokens += round(len(enc.encode(user))*2) outputTotalTokens += round(len(enc.encode(user)) * 2)
return [inputTotalTokens, outputTotalTokens] return [inputTotalTokens, outputTotalTokens]
def combineList(tlist, text): def combineList(tlist, text):
if isinstance(text, list): if isinstance(text, list):
return [t for sublist in tlist for t in sublist] return [t for sublist in tlist for t in sublist]
return tlist[0] return tlist[0]
@retry(exceptions=Exception, tries=5, delay=5) @retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag, pbar, filename): def translateGPT(text, history, fullPromptFlag, pbar, filename):
mismatch = False mismatch = False
@ -510,8 +568,12 @@ def translateGPT(text, history, fullPromptFlag, pbar, filename):
for index, tItem in enumerate(tList): for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting # Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list): if isinstance(tItem, list):
payload = '\n'.join([f'`<Line{i}>{item}</Line{i}>`' for i, item in enumerate(tItem)]) payload = "\n".join(
payload = re.sub(r'(<Line\d+)(><)(\/Line\d+>)', r'\1>Placeholder Text<\3', payload) [f"`<Line{i}>{item}</Line{i}>`" for i, item in enumerate(tItem)]
)
payload = re.sub(
r"(<Line\d+)(><)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload
)
varResponse = subVars(payload) varResponse = subVars(payload)
subbedT = varResponse[0] subbedT = varResponse[0]
else: else:
@ -519,7 +581,7 @@ def translateGPT(text, history, fullPromptFlag, pbar, filename):
subbedT = varResponse[0] subbedT = varResponse[0]
# Things to Check before starting translation # Things to Check before starting translation
if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT): if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue continue
# Create Message # Create Message

1851
modules/rpgmakerace.py
View file

File diff suppressed because it is too large Load diff

1900
modules/rpgmakermvmz.py
View file

File diff suppressed because it is too large Load diff

323
modules/rpgmakerplugin.py
View file

@ -16,34 +16,34 @@ from tqdm import tqdm
# Open AI # Open AI
load_dotenv() load_dotenv()
if os.getenv('api').replace(' ', '') != '': if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv('api') openai.base_url = os.getenv("api")
openai.organization = os.getenv('org') openai.organization = os.getenv("org")
openai.api_key = os.getenv('key') openai.api_key = os.getenv("key")
#Globals # Globals
MODEL = os.getenv('model') MODEL = os.getenv("model")
TIMEOUT = int(os.getenv('timeout')) TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv('language').capitalize() LANGUAGE = os.getenv("language").capitalize()
PROMPT = Path('prompt.txt').read_text(encoding='utf-8') PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path('vocab.txt').read_text(encoding='utf-8') VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv('threads')) THREADS = int(os.getenv("threads"))
LOCK = threading.Lock() LOCK = threading.Lock()
WIDTH = int(os.getenv('width')) WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv('listWidth')) LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70 NOTEWIDTH = 70
MAXHISTORY = 10 MAXHISTORY = 10
ESTIMATE = '' ESTIMATE = ""
TOKENS = [0, 0] TOKENS = [0, 0]
NAMESLIST = [] NAMESLIST = []
NAMES = False # Output a list of all the character names found NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = True # Overwrites textwrap FIXTEXTWRAP = True # Overwrites textwrap
IGNORETLTEXT = False # Ignores all translated text. IGNORETLTEXT = False # Ignores all translated text.
MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
#tqdm Globals # tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0 POSITION = 0
LEAVE = False LEAVE = False
PBAR = None PBAR = None
@ -51,15 +51,16 @@ PBAR = None
# Pricing - Depends on the model https://openai.com/pricing # Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request # Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size. # If you are getting a MISMATCH LENGTH error, lower the batch size.
if 'gpt-3.5' in MODEL: if "gpt-3.5" in MODEL:
INPUTAPICOST = .002 INPUTAPICOST = 0.002
OUTPUTAPICOST = .002 OUTPUTAPICOST = 0.002
BATCHSIZE = 10 BATCHSIZE = 10
elif 'gpt-4' in MODEL: elif "gpt-4" in MODEL:
INPUTAPICOST = .005 INPUTAPICOST = 0.005
OUTPUTAPICOST = .015 OUTPUTAPICOST = 0.015
BATCHSIZE = 40 BATCHSIZE = 40
def handlePlugin(filename, estimate): def handlePlugin(filename, estimate):
global ESTIMATE, PBAR global ESTIMATE, PBAR
ESTIMATE = estimate ESTIMATE = estimate
@ -76,17 +77,21 @@ def handlePlugin(filename, estimate):
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
# Print Total # Print Total
totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL') totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps # Print any errors on maps
if len(MISMATCH) > 0: if len(MISMATCH) > 0:
return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET return (
totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
)
else: else:
return totalString return totalString
else: else:
try: try:
with open('translated/' + filename, 'w', encoding='utf_8', errors='ignore') as outFile: with open(
"translated/" + filename, "w", encoding="utf_8", errors="ignore"
) as outFile:
start = time.time() start = time.time()
translatedData = openFiles(filename) translatedData = openFiles(filename)
@ -99,23 +104,36 @@ def handlePlugin(filename, estimate):
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
except Exception: except Exception:
traceback.print_exc() traceback.print_exc()
return 'Fail' return "Fail"
return getResultString(["", TOKENS, None], end - start, "TOTAL")
return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def getResultString(translatedData, translationTime, filename): def getResultString(translatedData, translationTime, filename):
# File Print String # File Print String
totalTokenstring =\ totalTokenstring = (
Fore.YELLOW +\ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
'[Input: ' + str(translatedData[1][0]) + ']'\ "[Output: "
'[Output: ' + str(translatedData[1][1]) + ']'\ + str(translatedData[1][1])
'[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\ + "]" "[Cost: ${:,.4f}".format(
(translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']' (translatedData[1][0] * 0.001 * INPUTAPICOST)
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
)
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None: if translatedData[2] == None:
# Success # Success
return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET return (
filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.GREEN
+ " \u2713 "
+ Fore.RESET
)
else: else:
# Fail # Fail
@ -124,31 +142,41 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
errorString = str(e) + Fore.RED errorString = str(e) + Fore.RED
return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\ return (
errorString + Fore.RESET filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.RED
+ " \u2717 "
+ errorString
+ Fore.RESET
)
def openFiles(filename): def openFiles(filename):
with open('files/' + filename, 'r', encoding='utf_8') as readFile: with open("files/" + filename, "r", encoding="utf_8") as readFile:
translatedData = parsePlugin(readFile, filename) translatedData = parsePlugin(readFile, filename)
# Delete lines marked for deletion # Delete lines marked for deletion
finalData = [] finalData = []
for line in translatedData[0]: for line in translatedData[0]:
if line != '\\d\n': if line != "\\d\n":
finalData.append(line) finalData.append(line)
translatedData[0] = finalData translatedData[0] = finalData
return translatedData return translatedData
def parsePlugin(readFile, filename): def parsePlugin(readFile, filename):
totalTokens = [0,0] totalTokens = [0, 0]
# Read File into data # Read File into data
data = readFile.readlines() data = readFile.readlines()
# Create Progress Bar # Create Progress Bar
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
pbar.desc=filename pbar.desc = filename
try: try:
result = translatePlugin(data, pbar, filename, []) result = translatePlugin(data, pbar, filename, [])
@ -159,19 +187,20 @@ def parsePlugin(readFile, filename):
return [data, totalTokens, e] return [data, totalTokens, e]
return [data, totalTokens, None] return [data, totalTokens, None]
def translatePlugin(data, pbar, filename, translatedList): def translatePlugin(data, pbar, filename, translatedList):
stringList = [] stringList = []
currentGroup = [] currentGroup = []
tokens = [0,0] tokens = [0, 0]
speaker = '' speaker = ""
voice = False voice = False
global LOCK, ESTIMATE global LOCK, ESTIMATE
i = 0 i = 0
while i < len(data): while i < len(data):
voice = False voice = False
speaker = '' speaker = ""
newline = r'\n' newline = r"\n"
""" """
Plugin List Plugin List
@ -192,16 +221,16 @@ def translatePlugin(data, pbar, filename, translatedList):
for match in matchList: for match in matchList:
# Save Original String # Save Original String
originalString = match originalString = match
# Remove any textwrap # Remove any textwrap
match = match.replace(newline, ' ') match = match.replace(newline, " ")
# Pass 1 # Pass 1
if translatedList == []: if translatedList == []:
# Add String # Add String
if match != '\\\\\\\\': if match != "\\\\\\\\":
stringList.append(match.strip()) stringList.append(match.strip())
# Pass 2 # Pass 2
else: else:
# Get Text # Get Text
@ -216,7 +245,7 @@ def translatePlugin(data, pbar, filename, translatedList):
# Textwrap # Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH) translatedText = textwrap.fill(translatedText, width=WIDTH)
translatedText = translatedText.replace('\n', newline) translatedText = translatedText.replace("\n", newline)
# Replace Single Quotes # Replace Single Quotes
translatedText = translatedText.replace("'", "\\'") translatedText = translatedText.replace("'", "\\'")
@ -224,7 +253,7 @@ def translatePlugin(data, pbar, filename, translatedList):
# Set Data # Set Data
data[i] = data[i].replace(originalString, translatedText) data[i] = data[i].replace(originalString, translatedText)
# Next Line # Next Line
i += 1 i += 1
# EOF # EOF
@ -233,9 +262,9 @@ def translatePlugin(data, pbar, filename, translatedList):
pbar.total = len(stringList) pbar.total = len(stringList)
pbar.refresh() pbar.refresh()
PBAR = pbar PBAR = pbar
# Translate # Translate
response = translateGPT(stringList, '', True) response = translateGPT(stringList, "", True)
tokens[0] += response[1][0] tokens[0] += response[1][0]
tokens[1] += response[1][1] tokens[1] += response[1][1]
translatedList = response[0] translatedList = response[0]
@ -251,23 +280,32 @@ def translatePlugin(data, pbar, filename, translatedList):
MISMATCH.append(filename) MISMATCH.append(filename)
return tokens return tokens
# Save some money and enter the character before translation # Save some money and enter the character before translation
def getSpeaker(speaker): def getSpeaker(speaker):
match speaker: match speaker:
case 'ファイン': case "ファイン":
return ['Fine', [0,0]] return ["Fine", [0, 0]]
case '': case "":
return ['', [0,0]] return ["", [0, 0]]
case _: case _:
# Store Speaker # Store Speaker
if speaker not in str(NAMESLIST): if speaker not in str(NAMESLIST):
response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False) response = translateGPT(
speaker,
"Reply with the " + LANGUAGE + " translation of the NPC name.",
False,
)
response[0] = response[0].title() response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s") response[0] = response[0].replace("'S", "'s")
# Retry if name doesn't translate for some reason # Retry if name doesn't translate for some reason
if re.search(r'([a-zA-Z？?])', response[0]) == None: if re.search(r"([a-zA-Z？?])", response[0]) == None:
response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False) response = translateGPT(
speaker,
"Reply with the " + LANGUAGE + " translation of the NPC name.",
False,
)
response[0] = response[0].title() response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s") response[0] = response[0].replace("'S", "'s")
@ -278,74 +316,76 @@ def getSpeaker(speaker):
else: else:
for i in range(len(NAMESLIST)): for i in range(len(NAMESLIST)):
if speaker == NAMESLIST[i][0]: if speaker == NAMESLIST[i][0]:
return [NAMESLIST[i][1],[0,0]] return [NAMESLIST[i][1], [0, 0]]
return [speaker,[0,0]] return [speaker, [0, 0]]
def subVars(jaString): def subVars(jaString):
jaString = jaString.replace('\u3000', ' ') jaString = jaString.replace("\u3000", " ")
# Nested # Nested
count = 0 count = 0
nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString) nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList) nestedList = set(nestedList)
if len(nestedList) != 0: if len(nestedList) != 0:
for icon in nestedList: for icon in nestedList:
jaString = jaString.replace(icon, '[Nested_' + str(count) + ']') jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString) iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList) iconList = set(iconList)
if len(iconList) != 0: if len(iconList) != 0:
for icon in iconList: for icon in iconList:
jaString = jaString.replace(icon, '[Ascii_' + str(count) + ']') jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
colorList = re.findall(r'([\\]+c\[\d+\][\\]+c|[\\]+c\[\d+\])', jaString) colorList = re.findall(r"([\\]+c\[\d+\][\\]+c|[\\]+c\[\d+\])", jaString)
colorList = set(colorList) colorList = set(colorList)
if len(colorList) != 0: if len(colorList) != 0:
for color in colorList: for color in colorList:
jaString = jaString.replace(color, '[Color_' + str(count) + ']') jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1 count += 1
# Names # Names
count = 0 count = 0
nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString) nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList) nameList = set(nameList)
if len(nameList) != 0: if len(nameList) != 0:
for name in nameList: for name in nameList:
jaString = jaString.replace(name, '[Noun_' + str(count) + ']') jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1 count += 1
# Variables # Variables
count = 0 count = 0
varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString) varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList) varList = set(varList)
if len(varList) != 0: if len(varList) != 0:
for var in varList: for var in varList:
jaString = jaString.replace(var, '[Var_' + str(count) + ']') jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
formatList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString) formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList) formatList = set(formatList)
if len(formatList) != 0: if len(formatList) != 0:
for var in formatList: for var in formatList:
jaString = jaString.replace(var, '[FCode_' + str(count) + ']') jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1 count += 1
# Put all lists in list and return # Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList] allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList] return [jaString, allList]
def resubVars(translatedText, allList): def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense # Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r'\[\s?.+?\s?\]', translatedText) matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0: if len(matchList) > 0:
for match in matchList: for match in matchList:
text = match.strip() text = match.strip()
@ -355,54 +395,58 @@ def resubVars(translatedText, allList):
count = 0 count = 0
if len(allList[0]) != 0: if len(allList[0]) != 0:
for var in allList[0]: for var in allList[0]:
translatedText = translatedText.replace('[Nested_' + str(count) + ']', var) translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
if len(allList[1]) != 0: if len(allList[1]) != 0:
for var in allList[1]: for var in allList[1]:
translatedText = translatedText.replace('[Ascii_' + str(count) + ']', var) translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
if len(allList[2]) != 0: if len(allList[2]) != 0:
for var in allList[2]: for var in allList[2]:
translatedText = translatedText.replace('[Color_' + str(count) + ']', var) translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1 count += 1
# Names # Names
count = 0 count = 0
if len(allList[3]) != 0: if len(allList[3]) != 0:
for var in allList[3]: for var in allList[3]:
translatedText = translatedText.replace('[Noun_' + str(count) + ']', var) translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1 count += 1
# Vars # Vars
count = 0 count = 0
if len(allList[4]) != 0: if len(allList[4]) != 0:
for var in allList[4]: for var in allList[4]:
translatedText = translatedText.replace('[Var_' + str(count) + ']', var) translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
if len(allList[5]) != 0: if len(allList[5]) != 0:
for var in allList[5]: for var in allList[5]:
translatedText = translatedText.replace('[FCode_' + str(count) + ']', var) translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1 count += 1
return translatedText return translatedText
def batchList(input_list, batch_size): def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0: if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer") raise ValueError("batch_size must be a positive integer")
return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)] return [
input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
]
def createContext(fullPromptFlag, subbedT): def createContext(fullPromptFlag, subbedT):
characters = 'Game Characters:\n\ characters = "Game Characters:\n\
シェーア (Shea) - Female\n\ シェーア (Shea) - Female\n\
ミューテ (Mute) - Female\n\ ミューテ (Mute) - Female\n\
タビノ (Tabino) - Female\n\ タビノ (Tabino) - Female\n\
@ -411,10 +455,12 @@ def createContext(fullPromptFlag, subbedT):
ソフィー (Sophie) - Female\n\ ソフィー (Sophie) - Female\n\
ドーラ (Dora) - Female\n\ ドーラ (Dora) - Female\n\
ミューレ (Mule) - Female\n\ ミューレ (Mule) - Female\n\
' "
system = PROMPT + VOCAB if fullPromptFlag else \ system = (
f"\ PROMPT + VOCAB
if fullPromptFlag
else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\ - You translate everything, including erotic and explicit content.\n\
@ -426,12 +472,14 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\ {VOCAB}\n\
" "
)
if isinstance(subbedT, list): if isinstance(subbedT, list):
user = f'```json\n{subbedT}```' user = f"```json\n{subbedT}```"
else: else:
user = subbedT user = subbedT
return characters, system, user return characters, system, user
def translateText(characters, system, user, history, penalty, format): def translateText(characters, system, user, history, penalty, format):
# Prompt # Prompt
msg = [{"role": "system", "content": system + characters}] msg = [{"role": "system", "content": system + characters}]
@ -446,13 +494,13 @@ def translateText(characters, system, user, history, penalty, format):
msg.append({"role": "system", "content": history}) msg.append({"role": "system", "content": history})
# Response Format # Response Format
if format == 'json': if format == "json":
responseFormat = { "type": "json_object" } responseFormat = {"type": "json_object"}
else: else:
responseFormat = { "type": "text" } responseFormat = {"type": "text"}
# Content to TL # Content to TL
msg.append({"role": "user", "content": f'{user}'}) msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create( response = openai.chat.completions.create(
temperature=0, temperature=0,
frequency_penalty=penalty, frequency_penalty=penalty,
@ -462,18 +510,19 @@ def translateText(characters, system, user, history, penalty, format):
) )
return response return response
def cleanTranslatedText(translatedText, varResponse): def cleanTranslatedText(translatedText, varResponse):
placeholders = { placeholders = {
f'{LANGUAGE} Translation: ': '', f"{LANGUAGE} Translation: ": "",
'Translation: ': '', "Translation: ": "",
'っ': '', "っ": "",
'〜': '~', "〜": "~",
'ッ': '', "ッ": "",
'。': '.', "。": ".",
'「': '\\"', "「": '\\"',
'」': '\\"', "」": '\\"',
'- ': '-', "- ": "-",
'Placeholder Text': '', "Placeholder Text": "",
# Add more replacements as needed # Add more replacements as needed
} }
for target, replacement in placeholders.items(): for target, replacement in placeholders.items():
@ -484,11 +533,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1]) translatedText = resubVars(translatedText, varResponse[1])
return translatedText return translatedText
def elongateCharacters(text): def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters # Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character # Using a positive lookbehind assertion to capture the preceding character
pattern = r'(?<=(.))ー+' pattern = r"(?<=(.))ー+"
# Define a replacement function that elongates the captured character # Define a replacement function that elongates the captured character
def repl(match): def repl(match):
char = match.group(1) # The character before the ー sequence char = match.group(1) # The character before the ー sequence
@ -498,6 +548,7 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text # Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text) return re.sub(pattern, repl, text)
def extractTranslation(translatedTextList, is_list): def extractTranslation(translatedTextList, is_list):
try: try:
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList) translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
@ -510,15 +561,15 @@ def extractTranslation(translatedTextList, is_list):
return string_list[0] return string_list[0]
except Exception as e: except Exception as e:
PBAR.write(f'extractTranslation Error: {e} on String {translatedTextList}') PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
return None return None
def countTokens(characters, system, user, history): def countTokens(characters, system, user, history):
inputTotalTokens = 0 inputTotalTokens = 0
outputTotalTokens = 0 outputTotalTokens = 0
enc = tiktoken.encoding_for_model('gpt-4') enc = tiktoken.encoding_for_model("gpt-4")
# Input # Input
if isinstance(history, list): if isinstance(history, list):
for line in history: for line in history:
@ -530,26 +581,28 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user)) inputTotalTokens += len(enc.encode(user))
# Output # Output
outputTotalTokens += round(len(enc.encode(user))*3) outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens] return [inputTotalTokens, outputTotalTokens]
def combineList(tlist, text): def combineList(tlist, text):
if isinstance(text, list): if isinstance(text, list):
return [t for sublist in tlist for t in sublist] return [t for sublist in tlist for t in sublist]
return tlist[0] return tlist[0]
@retry(exceptions=Exception, tries=5, delay=5) @retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag): def translateGPT(text, history, fullPromptFlag):
global PBAR global PBAR
mismatch = False mismatch = False
totalTokens = [0, 0] totalTokens = [0, 0]
if isinstance(text, list): if isinstance(text, list):
format = 'json' format = "json"
tList = batchList(text, BATCHSIZE) tList = batchList(text, BATCHSIZE)
else: else:
format = 'text' format = "text"
tList = [text] tList = [text]
for index, tItem in enumerate(tList): for index, tItem in enumerate(tList):
@ -564,7 +617,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0] subbedT = varResponse[0]
# Things to Check before starting translation # Things to Check before starting translation
if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT): if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
if PBAR is not None: if PBAR is not None:
PBAR.update(len(tItem)) PBAR.update(len(tItem))
continue continue
@ -589,9 +642,13 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse) translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list): if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True) extractedTranslations = extractTranslation(translatedText, True)
if extractedTranslations == None or len(tItem) != len(extractedTranslations): if extractedTranslations == None or len(tItem) != len(
extractedTranslations
):
# Mismatch. Try Again # Mismatch. Try Again
response = translateText(characters, system, user, history, 0.05, format) response = translateText(
characters, system, user, history, 0.05, format
)
translatedText = response.choices[0].message.content translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens totalTokens[1] += response.usage.completion_tokens
@ -600,13 +657,17 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse) translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list): if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True) extractedTranslations = extractTranslation(translatedText, True)
if extractedTranslations == None or len(tItem) != len(extractedTranslations): if extractedTranslations == None or len(tItem) != len(
mismatch = True # Just here for breakpoint extractedTranslations
):
mismatch = True # Just here for breakpoint
# Set if no mismatch # Set if no mismatch
if mismatch == False: if mismatch == False:
tList[index] = extractedTranslations tList[index] = extractedTranslations
history = extractedTranslations[-10:] # Update history if we have a list history = extractedTranslations[
-10:
] # Update history if we have a list
else: else:
history = text[-10:] history = text[-10:]
mismatch = False mismatch = False
@ -617,7 +678,7 @@ def translateGPT(text, history, fullPromptFlag):
PBAR.update(len(tItem)) PBAR.update(len(tItem))
else: else:
# Ensure we're passing a single string to extractTranslation # Ensure we're passing a single string to extractTranslation
tList[index] = translatedText tList[index] = translatedText
finalList = combineList(tList, text) finalList = combineList(tList, text)
return [finalList, totalTokens] return [finalList, totalTokens]

46
modules/sakuranbo.py
View file

@ -16,7 +16,7 @@ from tqdm import tqdm
# Open AI # Open AI
load_dotenv() load_dotenv()
if os.getenv("api").replace(" ", "") != "": if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv('api') openai.base_url = os.getenv("api")
openai.organization = os.getenv("org") openai.organization = os.getenv("org")
openai.api_key = os.getenv("key") openai.api_key = os.getenv("key")
@ -95,8 +95,9 @@ def getResultString(translatedData, translationTime, filename):
# File Print String # File Print String
totalTokenstring = ( totalTokenstring = (
Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]" Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
"[Output: " + str(translatedData[1][1]) + "]" "[Output: "
"[Cost: ${:,.4f}".format( + str(translatedData[1][1])
+ "]" "[Cost: ${:,.4f}".format(
(translatedData[1][0] * 0.001 * INPUTAPICOST) (translatedData[1][0] * 0.001 * INPUTAPICOST)
+ (translatedData[1][1] * 0.001 * OUTPUTAPICOST) + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
) )
@ -189,17 +190,17 @@ def translateTyrano(data, pbar):
if syncIndex > i: if syncIndex > i:
i = syncIndex i = syncIndex
if '[▼]' in data[i]: if "[▼]" in data[i]:
data[i] = data[i].replace('[▼]'.strip(), '[page]\n') data[i] = data[i].replace("[▼]".strip(), "[page]\n")
# If there isn't any Japanese in the text just skip # If there isn't any Japanese in the text just skip
if IGNORETLTEXT is True: if IGNORETLTEXT is True:
if not re.search(r'[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+', data[i]): if not re.search(r"[一-龠]+|[ぁ-ゔ]+|[ァ-ヴー]+", data[i]):
# Keep textHistory list at length maxHistory # Keep textHistory list at length maxHistory
textHistory.append('\"' + data[i] + '\"') textHistory.append('"' + data[i] + '"')
if len(textHistory) > maxHistory: if len(textHistory) > maxHistory:
textHistory.pop(0) textHistory.pop(0)
currentGroup = [] currentGroup = []
continue continue
# Speaker # Speaker
@ -215,18 +216,16 @@ def translateTyrano(data, pbar):
speaker = "Narrator" speaker = "Narrator"
elif "マコ" in matchList[0]: elif "マコ" in matchList[0]:
speaker = "Mako" speaker = "Mako"
elif '少年' in matchList[0]: elif "少年" in matchList[0]:
speaker = "Boy" speaker = "Boy"
elif '友達' in matchList[0]: elif "友達" in matchList[0]:
speaker = "Friend" speaker = "Friend"
elif '少女' in matchList[0]: elif "少女" in matchList[0]:
speaker = "Girl" speaker = "Girl"
else: else:
response = translateGPT( response = translateGPT(
matchList[0], matchList[0],
"Reply with only the " "Reply with only the " + LANGUAGE + " translation of the NPC name",
+ LANGUAGE
+ " translation of the NPC name",
True, True,
) )
speaker = response[0] speaker = response[0]
@ -263,13 +262,17 @@ def translateTyrano(data, pbar):
# Set Data # Set Data
translatedText = data[i].replace( translatedText = data[i].replace(
matchList[0], translatedText.replace(" ", "\u00A0") matchList[0], translatedText.replace(" ", "\u00a0")
) )
data[i] = translatedText data[i] = translatedText
# Grab Lines # Grab Lines
matchList = re.findall(r"^([^\n;@*\{\[].+[^;'{}\[]$)", data[i]) matchList = re.findall(r"^([^\n;@*\{\[].+[^;'{}\[]$)", data[i])
if len(matchList) > 0 and (re.search(r'^\[(.+)\sstorage=.+\],', data[i-1]) or re.search(r'^\[(.+)\]$', data[i-1]) or re.search(r'^《(.+)》', data[i-1])): if len(matchList) > 0 and (
re.search(r"^\[(.+)\sstorage=.+\],", data[i - 1])
or re.search(r"^\[(.+)\]$", data[i - 1])
or re.search(r"^《(.+)》", data[i - 1])
):
currentGroup.append(matchList[0]) currentGroup.append(matchList[0])
if len(data) > i + 1: if len(data) > i + 1:
matchList = re.findall(r"^([^\n;@*\{\[].+[^;'{}\[]$)", data[i + 1]) matchList = re.findall(r"^([^\n;@*\{\[].+[^;'{}\[]$)", data[i + 1])
@ -323,10 +326,10 @@ def translateTyrano(data, pbar):
# Set # Set
if delFlag is True: if delFlag is True:
data.insert(i, translatedText.strip() + '\n') data.insert(i, translatedText.strip() + "\n")
delFlag = False delFlag = False
else: else:
data[i] = translatedText.strip() + '\n' data[i] = translatedText.strip() + "\n"
# Keep textHistory list at length maxHistory # Keep textHistory list at length maxHistory
if len(textHistory) > maxHistory: if len(textHistory) > maxHistory:
@ -399,10 +402,10 @@ def translateTyrano(data, pbar):
# Set # Set
if delFlag is True: if delFlag is True:
data.insert(i, translatedText.strip() + '\n') data.insert(i, translatedText.strip() + "\n")
delFlag = False delFlag = False
else: else:
data[i] = translatedText.strip() + '\n' data[i] = translatedText.strip() + "\n"
# Keep textHistory list at length maxHistory # Keep textHistory list at length maxHistory
if len(textHistory) > maxHistory: if len(textHistory) > maxHistory:
@ -418,6 +421,7 @@ def translateTyrano(data, pbar):
return tokens return tokens
def subVars(jaString): def subVars(jaString):
jaString = jaString.replace("\u3000", " ") jaString = jaString.replace("\u3000", " ")
@ -551,7 +555,7 @@ def translateGPT(t, history, fullPromptFlag):
# If ESTIMATE is True just count this as an execution and return. # If ESTIMATE is True just count this as an execution and return.
if ESTIMATE: if ESTIMATE:
enc = tiktoken.encoding_for_model('gpt-4') enc = tiktoken.encoding_for_model("gpt-4")
historyRaw = "" historyRaw = ""
if isinstance(history, list): if isinstance(history, list):
for line in history: for line in history:

387
modules/tyrano.py
View file

@ -15,35 +15,35 @@ from tqdm import tqdm
# Open AI # Open AI
load_dotenv() load_dotenv()
if os.getenv('api').replace(' ', '') != '': if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv('api') openai.base_url = os.getenv("api")
openai.organization = os.getenv('org') openai.organization = os.getenv("org")
openai.api_key = os.getenv('key') openai.api_key = os.getenv("key")
#Globals # Globals
PBAR = None PBAR = None
MODEL = os.getenv('model') MODEL = os.getenv("model")
TIMEOUT = int(os.getenv('timeout')) TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv('language').capitalize() LANGUAGE = os.getenv("language").capitalize()
PROMPT = Path('prompt.txt').read_text(encoding='utf-8') PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path('vocab.txt').read_text(encoding='utf-8') VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv('threads')) THREADS = int(os.getenv("threads"))
LOCK = threading.Lock() LOCK = threading.Lock()
WIDTH = int(os.getenv('width')) WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv('listWidth')) LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70 NOTEWIDTH = 70
MAXHISTORY = 10 MAXHISTORY = 10
ESTIMATE = '' ESTIMATE = ""
TOKENS = [0, 0] TOKENS = [0, 0]
NAMESLIST = [] NAMESLIST = []
NAMES = False # Output a list of all the character names found NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = False # Overwrites textwrap FIXTEXTWRAP = False # Overwrites textwrap
IGNORETLTEXT = False # Ignores all translated text. IGNORETLTEXT = False # Ignores all translated text.
MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
#tqdm Globals # tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0 POSITION = 0
LEAVE = False LEAVE = False
@ -54,15 +54,16 @@ TEXTWRAPCHOICES = True
# Pricing - Depends on the model https://openai.com/pricing # Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request # Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size. # If you are getting a MISMATCH LENGTH error, lower the batch size.
if 'gpt-3.5' in MODEL: if "gpt-3.5" in MODEL:
INPUTAPICOST = .002 INPUTAPICOST = 0.002
OUTPUTAPICOST = .002 OUTPUTAPICOST = 0.002
BATCHSIZE = 10 BATCHSIZE = 10
elif 'gpt-4' in MODEL: elif "gpt-4" in MODEL:
INPUTAPICOST = .005 INPUTAPICOST = 0.005
OUTPUTAPICOST = .015 OUTPUTAPICOST = 0.015
BATCHSIZE = 40 BATCHSIZE = 40
def handleTyrano(filename, estimate): def handleTyrano(filename, estimate):
global ESTIMATE global ESTIMATE
ESTIMATE = estimate ESTIMATE = estimate
@ -79,17 +80,21 @@ def handleTyrano(filename, estimate):
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
# Print Total # Print Total
totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL') totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps # Print any errors on maps
if len(MISMATCH) > 0: if len(MISMATCH) > 0:
return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET return (
totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
)
else: else:
return totalString return totalString
else: else:
try: try:
with open('translated/' + filename, 'w', encoding='utf8', errors='ignore') as outFile: with open(
"translated/" + filename, "w", encoding="utf8", errors="ignore"
) as outFile:
start = time.time() start = time.time()
translatedData = openFiles(filename) translatedData = openFiles(filename)
@ -102,23 +107,36 @@ def handleTyrano(filename, estimate):
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
except Exception: except Exception:
traceback.print_exc() traceback.print_exc()
return 'Fail' return "Fail"
return getResultString(["", TOKENS, None], end - start, "TOTAL")
return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def getResultString(translatedData, translationTime, filename): def getResultString(translatedData, translationTime, filename):
# File Print String # File Print String
totalTokenstring =\ totalTokenstring = (
Fore.YELLOW +\ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
'[Input: ' + str(translatedData[1][0]) + ']'\ "[Output: "
'[Output: ' + str(translatedData[1][1]) + ']'\ + str(translatedData[1][1])
'[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\ + "]" "[Cost: ${:,.4f}".format(
(translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']' (translatedData[1][0] * 0.001 * INPUTAPICOST)
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
)
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None: if translatedData[2] == None:
# Success # Success
return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET return (
filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.GREEN
+ " \u2713 "
+ Fore.RESET
)
else: else:
# Fail # Fail
@ -127,34 +145,46 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
errorString = str(e) + Fore.RED errorString = str(e) + Fore.RED
return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\ return (
errorString + Fore.RESET filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.RED
+ " \u2717 "
+ errorString
+ Fore.RESET
)
def openFiles(filename): def openFiles(filename):
with open('files/' + filename, 'r', encoding='utf8') as readFile: with open("files/" + filename, "r", encoding="utf8") as readFile:
translatedData = parseTyrano(readFile, filename) translatedData = parseTyrano(readFile, filename)
# Delete lines marked for deletion # Delete lines marked for deletion
finalData = [] finalData = []
for line in translatedData[0]: for line in translatedData[0]:
if line != '\\d\n': if line != "\\d\n":
finalData.append(line) finalData.append(line)
translatedData[0] = finalData translatedData[0] = finalData
return translatedData return translatedData
def parseTyrano(readFile, filename): def parseTyrano(readFile, filename):
totalTokens = [0,0] totalTokens = [0, 0]
totalLines = 0 totalLines = 0
# Get total for progress bar # Get total for progress bar
data = readFile.readlines() data = readFile.readlines()
with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: with tqdm(
pbar.desc=filename bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE
) as pbar:
pbar.desc = filename
try: try:
result = translateTyrano(data, pbar, filename, False, [[],[]]) result = translateTyrano(data, pbar, filename, False, [[], []])
totalTokens[0] += result[0] totalTokens[0] += result[0]
totalTokens[1] += result[1] totalTokens[1] += result[1]
except Exception as e: except Exception as e:
@ -162,58 +192,63 @@ def parseTyrano(readFile, filename):
return [data, totalTokens, e] return [data, totalTokens, e]
return [data, totalTokens, None] return [data, totalTokens, None]
def translateTyrano(data, pbar, filename, setData, jobList): def translateTyrano(data, pbar, filename, setData, jobList):
textHistory = [] textHistory = []
lineList = jobList[0] lineList = jobList[0]
totalTokens = [0,0] totalTokens = [0, 0]
speaker = '' speaker = ""
global LOCK, ESTIMATE global LOCK, ESTIMATE
i = 0 i = 0
# Set Progress Bar # Set Progress Bar
global PBAR global PBAR
PBAR = pbar PBAR = pbar
while i < len(data): while i < len(data):
# Choices # Choices
choiceList = [] choiceList = []
choiceRegex = r'[sS]tatus.+?\](.+)' choiceRegex = r"[sS]tatus.+?\](.+)"
if 'tatus' in data[i]: if "tatus" in data[i]:
match = re.search(choiceRegex, data[i]) match = re.search(choiceRegex, data[i])
if match != None: if match != None:
jaString = match.group(1) jaString = match.group(1)
# Remove Textwrap # Remove Textwrap
if TEXTWRAPCHOICES is True: if TEXTWRAPCHOICES is True:
jaString = jaString.replace('[r]', ' ') jaString = jaString.replace("[r]", " ")
data[i] = data[i].replace('[r]', ' ') data[i] = data[i].replace("[r]", " ")
# Add to list # Add to list
choiceList.append(jaString) choiceList.append(jaString)
i += 1 i += 1
# Grab them all up for list # Grab them all up for list
while(i < len(data) and 'tatus' in data[i]): while i < len(data) and "tatus" in data[i]:
match = re.search(choiceRegex, data[i]) match = re.search(choiceRegex, data[i])
if match != None: if match != None:
jaString = match.group(1) jaString = match.group(1)
# Remove Textwrap # Remove Textwrap
if TEXTWRAPCHOICES is True: if TEXTWRAPCHOICES is True:
jaString = jaString.replace('[r]', ' ') jaString = jaString.replace("[r]", " ")
data[i] = data[i].replace('[r]', ' ') data[i] = data[i].replace("[r]", " ")
# Add to list # Add to list
choiceList.append(jaString) choiceList.append(jaString)
i += 1 i += 1
# Translate # Translate
if len(choiceList) != 0: if len(choiceList) != 0:
response = translateGPT(choiceList, 'Reply with the {LANGUAGE} translation of the text', True) response = translateGPT(
choiceList,
"Reply with the {LANGUAGE} translation of the text",
True,
)
choiceListTL = response[0] choiceListTL = response[0]
totalTokens[0] += response[1][0] totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1] totalTokens[1] += response[1][1]
# Set Data # Set Data
if len(choiceList) == len(choiceListTL): if len(choiceList) == len(choiceListTL):
i = i - len(choiceListTL) i = i - len(choiceListTL)
@ -223,75 +258,83 @@ def translateTyrano(data, pbar, filename, setData, jobList):
# Textwrap # Textwrap
if TEXTWRAPCHOICES is True: if TEXTWRAPCHOICES is True:
translatedText = textwrap.fill(translatedText, WIDTH) translatedText = textwrap.fill(translatedText, WIDTH)
translatedText = translatedText.replace('\n', '[r]') translatedText = translatedText.replace("\n", "[r]")
data[i] = data[i].replace(choiceList[j], translatedText) data[i] = data[i].replace(choiceList[j], translatedText)
i += 1 i += 1
else: else:
with LOCK: with LOCK:
if filename not in MISMATCH: if filename not in MISMATCH:
MISMATCH.append(filename) MISMATCH.append(filename)
if DIALOGUEFLAG is True: if DIALOGUEFLAG is True:
# Speaker # Speaker
if '[@]' in data[i]: if "[@]" in data[i]:
if 'FACE' not in data[i]: if "FACE" not in data[i]:
matchList = re.findall(r'\[(.*?)\].+\[.*\]', data[i]) matchList = re.findall(r"\[(.*?)\].+\[.*\]", data[i])
else: else:
matchList = re.findall(r'face=.+?\]\[(.+?)\]', data[i]) matchList = re.findall(r"face=.+?\]\[(.+?)\]", data[i])
if len(matchList) != 0 and '=' not in matchList[0] and re.search(r'\[.+\]', matchList[0]) == None: if (
len(matchList) != 0
and "=" not in matchList[0]
and re.search(r"\[.+\]", matchList[0]) == None
):
response = getSpeaker(matchList[0]) response = getSpeaker(matchList[0])
speaker = response[0] speaker = response[0]
totalTokens[0] += response[1][0] totalTokens[0] += response[1][0]
totalTokens[1] += response[1][1] totalTokens[1] += response[1][1]
# data[i] = data[i].replace(matchList[0], f'{speaker}') # data[i] = data[i].replace(matchList[0], f'{speaker}')
else: else:
speaker = '' speaker = ""
# Lines # Lines
if 'FACE' not in data[i]: if "FACE" not in data[i]:
matchList = re.findall(r'\[.+?\](.+)\[.+\]', data[i]) matchList = re.findall(r"\[.+?\](.+)\[.+\]", data[i])
else: else:
matchList = re.findall(r'face=.+?\]\[.+?\](.+)\[.+\]', data[i]) matchList = re.findall(r"face=.+?\]\[.+?\](.+)\[.+\]", data[i])
if len(matchList) > 0 and '=' not in matchList[0]: if len(matchList) > 0 and "=" not in matchList[0]:
# No Japanese text # No Japanese text
if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', matchList[0]): if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", matchList[0]):
i += 1 i += 1
continue continue
# Remove [r] and [l] # Remove [r] and [l]
oldjaString = matchList[0] oldjaString = matchList[0]
jaString = oldjaString jaString = oldjaString
jaString = jaString.replace('[r]', ' ') jaString = jaString.replace("[r]", " ")
jaString = jaString.replace('[l]', '') jaString = jaString.replace("[l]", "")
# Join up 401 groups for better translation. # Join up 401 groups for better translation.
finalJAString = jaString finalJAString = jaString
# Remove Extra Stuff bad for translation. # Remove Extra Stuff bad for translation.
finalJAString = finalJAString.replace('ﾞ', '') finalJAString = finalJAString.replace("ﾞ", "")
finalJAString = finalJAString.replace('・', '.') finalJAString = finalJAString.replace("・", ".")
finalJAString = finalJAString.replace('‶', '') finalJAString = finalJAString.replace("‶", "")
finalJAString = finalJAString.replace('”', '') finalJAString = finalJAString.replace("”", "")
finalJAString = finalJAString.replace('―', '-') finalJAString = finalJAString.replace("―", "-")
finalJAString = finalJAString.replace('…', '...') finalJAString = finalJAString.replace("…", "...")
finalJAString = re.sub(r'(\.{3}\.+)', '...', finalJAString) finalJAString = re.sub(r"(\.{3}\.+)", "...", finalJAString)
finalJAString = finalJAString.replace('　', ' ') finalJAString = finalJAString.replace("　", " ")
finalJAString = finalJAString.replace('】', ')') finalJAString = finalJAString.replace("】", ")")
finalJAString = finalJAString.replace('【　', '(') finalJAString = finalJAString.replace("【　", "(")
# Furigana Removal # Furigana Removal
matchList = re.findall(r'(\[ruby\stext=.+text=\"(.+)\"\])', finalJAString) matchList = re.findall(
r"(\[ruby\stext=.+text=\"(.+)\"\])", finalJAString
)
if len(matchList) > 0: if len(matchList) > 0:
finalJAString = finalJAString.replace(matchList[0][0], matchList[0][1]) finalJAString = finalJAString.replace(
matchList[0][0], matchList[0][1]
)
# Add Speaker (If there is one) # Add Speaker (If there is one)
if speaker != '': if speaker != "":
finalJAString = f'{speaker}: {finalJAString}' finalJAString = f"{speaker}: {finalJAString}"
# [Passthrough 1] Append To List # [Passthrough 1] Append To List
if setData is False: if setData is False:
lineList.append(finalJAString) lineList.append(finalJAString)
# [Passthrough 2] Set Data # [Passthrough 2] Set Data
else: else:
# Grab and Pop # Grab and Pop
@ -299,22 +342,24 @@ def translateTyrano(data, pbar, filename, setData, jobList):
lineList.pop(0) lineList.pop(0)
# Remove speaker # Remove speaker
translatedText = re.sub(r'^\[?(.+?)\]?\s?[|:]\s?', '', translatedText) translatedText = re.sub(
r"^\[?(.+?)\]?\s?[|:]\s?", "", translatedText
)
# Textwrap # Textwrap
translatedText = textwrap.fill(translatedText, WIDTH) translatedText = textwrap.fill(translatedText, WIDTH)
translatedText = translatedText.replace('\n', '[r]') translatedText = translatedText.replace("\n", "[r]")
# Set Data # Set Data
data[i] = data[i].replace(oldjaString, translatedText) data[i] = data[i].replace(oldjaString, translatedText)
# Next Line # Next Line
i += 1 i += 1
# Translate Data # Translate Data
lineListTL = [] lineListTL = []
setData = False setData = False
# Line List # Line List
if len(lineList) > 0: if len(lineList) > 0:
pbar.total = len(lineList) pbar.total = len(lineList)
@ -335,17 +380,23 @@ def translateTyrano(data, pbar, filename, setData, jobList):
translateTyrano(data, pbar, filename, True, [lineListTL]) translateTyrano(data, pbar, filename, True, [lineListTL])
return totalTokens return totalTokens
# Save some money and enter the character before translation # Save some money and enter the character before translation
def getSpeaker(speaker): def getSpeaker(speaker):
match speaker: match speaker:
case 'ファイン': case "ファイン":
return ['Fine', [0,0]] return ["Fine", [0, 0]]
case '': case "":
return ['', [0,0]] return ["", [0, 0]]
case _: case _:
# Store Speaker # Store Speaker
if speaker not in str(NAMESLIST): if speaker not in str(NAMESLIST):
response = translateGPT(speaker, 'Reply with only the '+ LANGUAGE +' translation of the NPC name.', False) response = translateGPT(
speaker,
"Reply with only the " + LANGUAGE + " translation of the NPC name.",
False,
)
response[0] = response[0].title() response[0] = response[0].title()
speakerList = [speaker, response[0]] speakerList = [speaker, response[0]]
NAMESLIST.append(speakerList) NAMESLIST.append(speakerList)
@ -354,74 +405,76 @@ def getSpeaker(speaker):
else: else:
for i in range(len(NAMESLIST)): for i in range(len(NAMESLIST)):
if speaker == NAMESLIST[i][0]: if speaker == NAMESLIST[i][0]:
return [NAMESLIST[i][1],[0,0]] return [NAMESLIST[i][1], [0, 0]]
return [speaker,[0,0]] return [speaker, [0, 0]]
def subVars(jaString): def subVars(jaString):
jaString = jaString.replace('\u3000', ' ') jaString = jaString.replace("\u3000", " ")
# Nested # Nested
count = 0 count = 0
nestedList = re.findall(r'[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]', jaString) nestedList = re.findall(r"[\\]+[\w]+\[[\\]+[\w]+\[[0-9]+\]\]", jaString)
nestedList = set(nestedList) nestedList = set(nestedList)
if len(nestedList) != 0: if len(nestedList) != 0:
for icon in nestedList: for icon in nestedList:
jaString = jaString.replace(icon, '[Nested_' + str(count) + ']') jaString = jaString.replace(icon, "[Nested_" + str(count) + "]")
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
iconList = re.findall(r'[\\]+[iIkKwWaA]+\[[0-9]+\]', jaString) iconList = re.findall(r"[\\]+[iIkKwWaA]+\[[0-9]+\]", jaString)
iconList = set(iconList) iconList = set(iconList)
if len(iconList) != 0: if len(iconList) != 0:
for icon in iconList: for icon in iconList:
jaString = jaString.replace(icon, '[Ascii_' + str(count) + ']') jaString = jaString.replace(icon, "[Ascii_" + str(count) + "]")
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
colorList = re.findall(r'[\\]+[cC]\[[0-9]+\]', jaString) colorList = re.findall(r"[\\]+[cC]\[[0-9]+\]", jaString)
colorList = set(colorList) colorList = set(colorList)
if len(colorList) != 0: if len(colorList) != 0:
for color in colorList: for color in colorList:
jaString = jaString.replace(color, '[Color_' + str(count) + ']') jaString = jaString.replace(color, "[Color_" + str(count) + "]")
count += 1 count += 1
# Names # Names
count = 0 count = 0
nameList = re.findall(r'[\\]+[nN]\[.+?\]+', jaString) nameList = re.findall(r"[\\]+[nN]\[.+?\]+", jaString)
nameList = set(nameList) nameList = set(nameList)
if len(nameList) != 0: if len(nameList) != 0:
for name in nameList: for name in nameList:
jaString = jaString.replace(name, '[Noun_' + str(count) + ']') jaString = jaString.replace(name, "[Noun_" + str(count) + "]")
count += 1 count += 1
# Variables # Variables
count = 0 count = 0
varList = re.findall(r'[\\]+[vV]\[[0-9]+\]', jaString) varList = re.findall(r"[\\]+[vV]\[[0-9]+\]", jaString)
varList = set(varList) varList = set(varList)
if len(varList) != 0: if len(varList) != 0:
for var in varList: for var in varList:
jaString = jaString.replace(var, '[Var_' + str(count) + ']') jaString = jaString.replace(var, "[Var_" + str(count) + "]")
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
formatList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString) formatList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
formatList = set(formatList) formatList = set(formatList)
if len(formatList) != 0: if len(formatList) != 0:
for var in formatList: for var in formatList:
jaString = jaString.replace(var, '[FCode_' + str(count) + ']') jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1 count += 1
# Put all lists in list and return # Put all lists in list and return
allList = [nestedList, iconList, colorList, nameList, varList, formatList] allList = [nestedList, iconList, colorList, nameList, varList, formatList]
return [jaString, allList] return [jaString, allList]
def resubVars(translatedText, allList): def resubVars(translatedText, allList):
# Fix Spacing and ChatGPT Nonsense # Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r'\[\s?.+?\s?\]', translatedText) matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0: if len(matchList) > 0:
for match in matchList: for match in matchList:
text = match.strip() text = match.strip()
@ -431,60 +484,66 @@ def resubVars(translatedText, allList):
count = 0 count = 0
if len(allList[0]) != 0: if len(allList[0]) != 0:
for var in allList[0]: for var in allList[0]:
translatedText = translatedText.replace('[Nested_' + str(count) + ']', var) translatedText = translatedText.replace("[Nested_" + str(count) + "]", var)
count += 1 count += 1
# Icons # Icons
count = 0 count = 0
if len(allList[1]) != 0: if len(allList[1]) != 0:
for var in allList[1]: for var in allList[1]:
translatedText = translatedText.replace('[Ascii_' + str(count) + ']', var) translatedText = translatedText.replace("[Ascii_" + str(count) + "]", var)
count += 1 count += 1
# Colors # Colors
count = 0 count = 0
if len(allList[2]) != 0: if len(allList[2]) != 0:
for var in allList[2]: for var in allList[2]:
translatedText = translatedText.replace('[Color_' + str(count) + ']', var) translatedText = translatedText.replace("[Color_" + str(count) + "]", var)
count += 1 count += 1
# Names # Names
count = 0 count = 0
if len(allList[3]) != 0: if len(allList[3]) != 0:
for var in allList[3]: for var in allList[3]:
translatedText = translatedText.replace('[Noun_' + str(count) + ']', var) translatedText = translatedText.replace("[Noun_" + str(count) + "]", var)
count += 1 count += 1
# Vars # Vars
count = 0 count = 0
if len(allList[4]) != 0: if len(allList[4]) != 0:
for var in allList[4]: for var in allList[4]:
translatedText = translatedText.replace('[Var_' + str(count) + ']', var) translatedText = translatedText.replace("[Var_" + str(count) + "]", var)
count += 1 count += 1
# Formatting # Formatting
count = 0 count = 0
if len(allList[5]) != 0: if len(allList[5]) != 0:
for var in allList[5]: for var in allList[5]:
translatedText = translatedText.replace('[FCode_' + str(count) + ']', var) translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1 count += 1
return translatedText return translatedText
def batchList(input_list, batch_size): def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0: if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer") raise ValueError("batch_size must be a positive integer")
return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)] return [
input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
]
def createContext(fullPromptFlag, subbedT): def createContext(fullPromptFlag, subbedT):
characters = 'Game Characters:\n\ characters = "Game Characters:\n\
眠り姫 (Sleeping Princess) - Female\n\ 眠り姫 (Sleeping Princess) - Female\n\
迷子 (Lost Child) - Male\n\ 迷子 (Lost Child) - Male\n\
' "
system = PROMPT + VOCAB if fullPromptFlag else \ system = (
f"\ PROMPT + VOCAB
if fullPromptFlag
else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\ - You translate everything, including erotic and explicit content.\n\
@ -496,9 +555,11 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\ {VOCAB}\n\
" "
user = f'{subbedT}' )
user = f"{subbedT}"
return characters, system, user return characters, system, user
def translateText(characters, system, user, history, penalty): def translateText(characters, system, user, history, penalty):
# Prompt # Prompt
msg = [{"role": "system", "content": system + characters}] msg = [{"role": "system", "content": system + characters}]
@ -511,9 +572,9 @@ def translateText(characters, system, user, history, penalty):
msg.extend([{"role": "system", "content": h} for h in history]) msg.extend([{"role": "system", "content": h} for h in history])
else: else:
msg.append({"role": "system", "content": history}) msg.append({"role": "system", "content": history})
# Content to TL # Content to TL
msg.append({"role": "user", "content": f'{user}'}) msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create( response = openai.chat.completions.create(
temperature=0, temperature=0,
frequency_penalty=penalty, frequency_penalty=penalty,
@ -522,17 +583,18 @@ def translateText(characters, system, user, history, penalty):
) )
return response return response
def cleanTranslatedText(translatedText, varResponse): def cleanTranslatedText(translatedText, varResponse):
placeholders = { placeholders = {
f'{LANGUAGE} Translation: ': '', f"{LANGUAGE} Translation: ": "",
'Translation: ': '', "Translation: ": "",
'っ': '', "っ": "",
'〜': '~', "〜": "~",
'ッ': '', "ッ": "",
'。': '.', "。": ".",
'Placeholder Text': '', "Placeholder Text": "",
'[' : '(', "[": "(",
']' : ')' "]": ")",
# Add more replacements as needed # Add more replacements as needed
} }
for target, replacement in placeholders.items(): for target, replacement in placeholders.items():
@ -543,11 +605,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1]) translatedText = resubVars(translatedText, varResponse[1])
return translatedText return translatedText
def elongateCharacters(text): def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters # Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character # Using a positive lookbehind assertion to capture the preceding character
pattern = r'(?<=(.))ー+' pattern = r"(?<=(.))ー+"
# Define a replacement function that elongates the captured character # Define a replacement function that elongates the captured character
def repl(match): def repl(match):
char = match.group(1) # The character before the ー sequence char = match.group(1) # The character before the ー sequence
@ -557,8 +620,9 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text # Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text) return re.sub(pattern, repl, text)
def extractTranslation(translatedTextList, is_list): def extractTranslation(translatedTextList, is_list):
pattern = r'`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?' pattern = r"`?<[Ll]ine\d+>([\\]*.*?[\\]*?)<\/?[Ll]ine\d+>`?"
# If it's a batch (i.e., list), extract with tags; otherwise, return the single item. # If it's a batch (i.e., list), extract with tags; otherwise, return the single item.
if is_list: if is_list:
matchList = re.findall(pattern, translatedTextList) matchList = re.findall(pattern, translatedTextList)
@ -567,11 +631,12 @@ def extractTranslation(translatedTextList, is_list):
matchList = re.findall(pattern, translatedTextList) matchList = re.findall(pattern, translatedTextList)
return matchList[0][0] if matchList else translatedTextList return matchList[0][0] if matchList else translatedTextList
def countTokens(characters, system, user, history): def countTokens(characters, system, user, history):
inputTotalTokens = 0 inputTotalTokens = 0
outputTotalTokens = 0 outputTotalTokens = 0
enc = tiktoken.encoding_for_model('gpt-4') enc = tiktoken.encoding_for_model("gpt-4")
# Input # Input
if isinstance(history, list): if isinstance(history, list):
for line in history: for line in history:
@ -583,15 +648,17 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user)) inputTotalTokens += len(enc.encode(user))
# Output # Output
outputTotalTokens += round(len(enc.encode(user))*3) outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens] return [inputTotalTokens, outputTotalTokens]
def combineList(tlist, text): def combineList(tlist, text):
if isinstance(text, list): if isinstance(text, list):
return [t for sublist in tlist for t in sublist] return [t for sublist in tlist for t in sublist]
return tlist[0] return tlist[0]
@retry(exceptions=Exception, tries=5, delay=5) @retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag): def translateGPT(text, history, fullPromptFlag):
global PBAR global PBAR
@ -605,8 +672,12 @@ def translateGPT(text, history, fullPromptFlag):
for index, tItem in enumerate(tList): for index, tItem in enumerate(tList):
# Before sending to translation, if we have a list of items, add the formatting # Before sending to translation, if we have a list of items, add the formatting
if isinstance(tItem, list): if isinstance(tItem, list):
payload = '\n'.join([f'`<Line{i}>{item}</Line{i}>`' for i, item in enumerate(tItem)]) payload = "\n".join(
payload = re.sub(r'(<Line\d+)(><)(\/Line\d+>)', r'\1>Placeholder Text<\3', payload) [f"`<Line{i}>{item}</Line{i}>`" for i, item in enumerate(tItem)]
)
payload = re.sub(
r"(<Line\d+)(><)(\/Line\d+>)", r"\1>Placeholder Text<\3", payload
)
varResponse = subVars(payload) varResponse = subVars(payload)
subbedT = varResponse[0] subbedT = varResponse[0]
else: else:
@ -614,7 +685,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0] subbedT = varResponse[0]
# Things to Check before starting translation # Things to Check before starting translation
if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT): if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
continue continue
# Create Message # Create Message
@ -651,11 +722,13 @@ def translateGPT(text, history, fullPromptFlag):
extractedTranslations = extractTranslation(translatedText, True) extractedTranslations = extractTranslation(translatedText, True)
tList[index] = extractedTranslations tList[index] = extractedTranslations
if len(tItem) != len(extractedTranslations): if len(tItem) != len(extractedTranslations):
mismatch = True # Just here for breakpoint mismatch = True # Just here for breakpoint
# Create History # Create History
if not mismatch: if not mismatch:
history = extractedTranslations[-10:] # Update history if we have a list history = extractedTranslations[
-10:
] # Update history if we have a list
else: else:
history = text[-10:] history = text[-10:]
PBAR.update(len(tItem)) PBAR.update(len(tItem))

1334
modules/wolf.py
View file

File diff suppressed because it is too large Load diff

331
modules/wolf2.py
View file

@ -16,49 +16,50 @@ from tqdm import tqdm
# Open AI # Open AI
load_dotenv() load_dotenv()
if os.getenv('api').replace(' ', '') != '': if os.getenv("api").replace(" ", "") != "":
openai.base_url = os.getenv('api') openai.base_url = os.getenv("api")
openai.organization = os.getenv('org') openai.organization = os.getenv("org")
openai.api_key = os.getenv('key') openai.api_key = os.getenv("key")
#Globals # Globals
MODEL = os.getenv('model') MODEL = os.getenv("model")
TIMEOUT = int(os.getenv('timeout')) TIMEOUT = int(os.getenv("timeout"))
LANGUAGE = os.getenv('language').capitalize() LANGUAGE = os.getenv("language").capitalize()
PROMPT = Path('prompt.txt').read_text(encoding='utf-8') PROMPT = Path("prompt.txt").read_text(encoding="utf-8")
VOCAB = Path('vocab.txt').read_text(encoding='utf-8') VOCAB = Path("vocab.txt").read_text(encoding="utf-8")
THREADS = int(os.getenv('threads')) THREADS = int(os.getenv("threads"))
LOCK = threading.Lock() LOCK = threading.Lock()
WIDTH = int(os.getenv('width')) WIDTH = int(os.getenv("width"))
LISTWIDTH = int(os.getenv('listWidth')) LISTWIDTH = int(os.getenv("listWidth"))
NOTEWIDTH = 70 NOTEWIDTH = 70
MAXHISTORY = 10 MAXHISTORY = 10
ESTIMATE = '' ESTIMATE = ""
TOKENS = [0, 0] TOKENS = [0, 0]
NAMESLIST = [] NAMESLIST = []
NAMES = False # Output a list of all the character names found NAMES = False # Output a list of all the character names found
BRFLAG = False # If the game uses <br> instead BRFLAG = False # If the game uses <br> instead
FIXTEXTWRAP = True # Overwrites textwrap FIXTEXTWRAP = True # Overwrites textwrap
IGNORETLTEXT = False # Ignores all translated text. IGNORETLTEXT = False # Ignores all translated text.
MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong) MISMATCH = [] # Lists files that throw a mismatch error (Length of GPT list response is wrong)
#tqdm Globals # tqdm Globals
BAR_FORMAT='{l_bar}{bar:10}{r_bar}{bar:-10b}' BAR_FORMAT = "{l_bar}{bar:10}{r_bar}{bar:-10b}"
POSITION = 0 POSITION = 0
LEAVE = False LEAVE = False
# Pricing - Depends on the model https://openai.com/pricing # Pricing - Depends on the model https://openai.com/pricing
# Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request # Batch Size - GPT 3.5 Struggles past 15 lines per request. GPT4 struggles past 50 lines per request
# If you are getting a MISMATCH LENGTH error, lower the batch size. # If you are getting a MISMATCH LENGTH error, lower the batch size.
if 'gpt-3.5' in MODEL: if "gpt-3.5" in MODEL:
INPUTAPICOST = .002 INPUTAPICOST = 0.002
OUTPUTAPICOST = .002 OUTPUTAPICOST = 0.002
BATCHSIZE = 10 BATCHSIZE = 10
elif 'gpt-4' in MODEL: elif "gpt-4" in MODEL:
INPUTAPICOST = .005 INPUTAPICOST = 0.005
OUTPUTAPICOST = .015 OUTPUTAPICOST = 0.015
BATCHSIZE = 40 BATCHSIZE = 40
def handleWOLF2(filename, estimate): def handleWOLF2(filename, estimate):
global ESTIMATE global ESTIMATE
ESTIMATE = estimate ESTIMATE = estimate
@ -75,17 +76,21 @@ def handleWOLF2(filename, estimate):
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
# Print Total # Print Total
totalString = getResultString(['', TOKENS, None], end - start, 'TOTAL') totalString = getResultString(["", TOKENS, None], end - start, "TOTAL")
# Print any errors on maps # Print any errors on maps
if len(MISMATCH) > 0: if len(MISMATCH) > 0:
return totalString + Fore.RED + f'\nMismatch Errors: {MISMATCH}' + Fore.RESET return (
totalString + Fore.RED + f"\nMismatch Errors: {MISMATCH}" + Fore.RESET
)
else: else:
return totalString return totalString
else: else:
try: try:
with open('translated/' + filename, 'w', encoding='shift_jis', errors='ignore') as outFile: with open(
"translated/" + filename, "w", encoding="shift_jis", errors="ignore"
) as outFile:
start = time.time() start = time.time()
translatedData = openFiles(filename) translatedData = openFiles(filename)
@ -98,23 +103,36 @@ def handleWOLF2(filename, estimate):
TOKENS[1] += translatedData[1][1] TOKENS[1] += translatedData[1][1]
except Exception: except Exception:
traceback.print_exc() traceback.print_exc()
return 'Fail' return "Fail"
return getResultString(["", TOKENS, None], end - start, "TOTAL")
return getResultString(['', TOKENS, None], end - start, 'TOTAL')
def getResultString(translatedData, translationTime, filename): def getResultString(translatedData, translationTime, filename):
# File Print String # File Print String
totalTokenstring =\ totalTokenstring = (
Fore.YELLOW +\ Fore.YELLOW + "[Input: " + str(translatedData[1][0]) + "]"
'[Input: ' + str(translatedData[1][0]) + ']'\ "[Output: "
'[Output: ' + str(translatedData[1][1]) + ']'\ + str(translatedData[1][1])
'[Cost: ${:,.4f}'.format((translatedData[1][0] * .001 * INPUTAPICOST) +\ + "]" "[Cost: ${:,.4f}".format(
(translatedData[1][1] * .001 * OUTPUTAPICOST)) + ']' (translatedData[1][0] * 0.001 * INPUTAPICOST)
timeString = Fore.BLUE + '[' + str(round(translationTime, 1)) + 's]' + (translatedData[1][1] * 0.001 * OUTPUTAPICOST)
)
+ "]"
)
timeString = Fore.BLUE + "[" + str(round(translationTime, 1)) + "s]"
if translatedData[2] == None: if translatedData[2] == None:
# Success # Success
return filename + ': ' + totalTokenstring + timeString + Fore.GREEN + u' \u2713 ' + Fore.RESET return (
filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.GREEN
+ " \u2713 "
+ Fore.RESET
)
else: else:
# Fail # Fail
@ -123,31 +141,41 @@ def getResultString(translatedData, translationTime, filename):
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
errorString = str(e) + Fore.RED errorString = str(e) + Fore.RED
return filename + ': ' + totalTokenstring + timeString + Fore.RED + u' \u2717 ' +\ return (
errorString + Fore.RESET filename
+ ": "
+ totalTokenstring
+ timeString
+ Fore.RED
+ " \u2717 "
+ errorString
+ Fore.RESET
)
def openFiles(filename): def openFiles(filename):
with open('files/' + filename, 'r', encoding='shift_jis') as readFile: with open("files/" + filename, "r", encoding="shift_jis") as readFile:
translatedData = parseWOLF(readFile, filename) translatedData = parseWOLF(readFile, filename)
# Delete lines marked for deletion # Delete lines marked for deletion
finalData = [] finalData = []
for line in translatedData[0]: for line in translatedData[0]:
if line != '\\d\n': if line != "\\d\n":
finalData.append(line) finalData.append(line)
translatedData[0] = finalData translatedData[0] = finalData
return translatedData return translatedData
def parseWOLF(readFile, filename): def parseWOLF(readFile, filename):
totalTokens = [0,0] totalTokens = [0, 0]
# Read File into data # Read File into data
data = readFile.readlines() data = readFile.readlines()
# Create Progress Bar # Create Progress Bar
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
pbar.desc=filename pbar.desc = filename
try: try:
result = translateWOLF(data, [], pbar, filename) result = translateWOLF(data, [], pbar, filename)
@ -158,39 +186,40 @@ def parseWOLF(readFile, filename):
return [data, totalTokens, e] return [data, totalTokens, e]
return [data, totalTokens, None] return [data, totalTokens, None]
def translateWOLF(data, translatedList, pbar, filename): def translateWOLF(data, translatedList, pbar, filename):
stringList = [] stringList = []
currentGroup = [] currentGroup = []
tokens = [0,0] tokens = [0, 0]
speaker = '' speaker = ""
global LOCK, ESTIMATE, PBAR global LOCK, ESTIMATE, PBAR
PBAR = pbar PBAR = pbar
i = 0 i = 0
while i < len(data): while i < len(data):
# Speaker # Speaker
matchList = re.findall(r'(.*)：', data[i]) matchList = re.findall(r"(.*)：", data[i])
if len(matchList) != 0: if len(matchList) != 0:
response = getSpeaker(matchList[0]) response = getSpeaker(matchList[0])
speaker = response[0] speaker = response[0]
tokens[0] += response[1][0] tokens[0] += response[1][0]
tokens[1] += response[1][1] tokens[1] += response[1][1]
data[i] = f'{speaker}：\n' data[i] = f"{speaker}：\n"
i += 1 i += 1
else: else:
speaker = '' speaker = ""
# Options # Options
if '//選択肢' in data[i]: if "//選択肢" in data[i]:
i += 1 i += 1
choiceList = [] choiceList = []
initialIndex = i initialIndex = i
while('//' in data[i] and 'の場合' not in data[i]): while "//" in data[i] and "の場合" not in data[i]:
choiceList.append(re.search(r'\/\/(.*)', data[i]).group(1)) choiceList.append(re.search(r"\/\/(.*)", data[i]).group(1))
i += 1 i += 1
# Translate # Translate
response = translateGPT(choiceList, 'This will be a dialogue option', True) response = translateGPT(choiceList, "This will be a dialogue option", True)
tokens[0] += response[1][0] tokens[0] += response[1][0]
tokens[1] += response[1][1] tokens[1] += response[1][1]
choiceListTL = response[0] choiceListTL = response[0]
@ -199,9 +228,9 @@ def translateWOLF(data, translatedList, pbar, filename):
if len(choiceList) == len(choiceListTL): if len(choiceList) == len(choiceListTL):
# Set Data # Set Data
i = initialIndex i = initialIndex
while('//' in data[i] and 'の場合' not in data[i]): while "//" in data[i] and "の場合" not in data[i]:
choiceListTL[0] = choiceListTL[0].replace(', ', '、') choiceListTL[0] = choiceListTL[0].replace(", ", "、")
data[i] = f'//{choiceListTL[0]}\n' data[i] = f"//{choiceListTL[0]}\n"
choiceListTL.pop(0) choiceListTL.pop(0)
i += 1 i += 1
@ -212,36 +241,46 @@ def translateWOLF(data, translatedList, pbar, filename):
MISMATCH.append(filename) MISMATCH.append(filename)
# Lines # Lines
if r'/' not in data[i] and '@' not in data[i] and data[i] != '\n': if r"/" not in data[i] and "@" not in data[i] and data[i] != "\n":
# Pass 1 # Pass 1
if translatedList == []: if translatedList == []:
# Grab Consecutive Strings # Grab Consecutive Strings
currentGroup.append(data[i]) currentGroup.append(data[i])
i += 1 i += 1
while i < len(data) and r'/' not in data[i] and '@' not in data[i] and data[i] != '\n': while (
i < len(data)
and r"/" not in data[i]
and "@" not in data[i]
and data[i] != "\n"
):
currentGroup.append(data[i]) currentGroup.append(data[i])
i += 1 i += 1
# Join up 401 groups for better translation. # Join up 401 groups for better translation.
if len(currentGroup) > 0: if len(currentGroup) > 0:
jaString = ''.join(currentGroup) jaString = "".join(currentGroup)
currentGroup = [] currentGroup = []
# Remove any textwrap # Remove any textwrap
jaString = jaString.replace('\n', ' ') jaString = jaString.replace("\n", " ")
# Add Speaker (If there is one) # Add Speaker (If there is one)
if speaker != '': if speaker != "":
jaString = f'{speaker}: {jaString}' jaString = f"{speaker}: {jaString}"
# Add String # Add String
stringList.append(jaString) stringList.append(jaString)
i += 1 i += 1
# Pass 2 # Pass 2
else: else:
# Insert Strings # Insert Strings
while i < len(data) and r'/' not in data[i] and '@' not in data[i] and data[i] != '\n': while (
i < len(data)
and r"/" not in data[i]
and "@" not in data[i]
and data[i] != "\n"
):
data.pop(i) data.pop(i)
# Get Text # Get Text
@ -252,13 +291,13 @@ def translateWOLF(data, translatedList, pbar, filename):
translatedList = None translatedList = None
# Remove added speaker # Remove added speaker
translatedText = re.sub(r'^.+?:\s', '', translatedText) translatedText = re.sub(r"^.+?:\s", "", translatedText)
# Textwrap # Textwrap
translatedText = textwrap.fill(translatedText, width=WIDTH) translatedText = textwrap.fill(translatedText, width=WIDTH)
# Set Data # Set Data
data.insert(i, f'{translatedText}\n') data.insert(i, f"{translatedText}\n")
i += 1 i += 1
# Nothing relevant. Skip Line. # Nothing relevant. Skip Line.
@ -270,9 +309,9 @@ def translateWOLF(data, translatedList, pbar, filename):
# Set Progress # Set Progress
pbar.total = len(stringList) pbar.total = len(stringList)
pbar.refresh() pbar.refresh()
# Translate # Translate
response = translateGPT(stringList, '', True) response = translateGPT(stringList, "", True)
tokens[0] += response[1][0] tokens[0] += response[1][0]
tokens[1] += response[1][1] tokens[1] += response[1][1]
translatedList = response[0] translatedList = response[0]
@ -288,23 +327,32 @@ def translateWOLF(data, translatedList, pbar, filename):
MISMATCH.append(filename) MISMATCH.append(filename)
return tokens return tokens
# Save some money and enter the character before translation # Save some money and enter the character before translation
def getSpeaker(speaker): def getSpeaker(speaker):
match speaker: match speaker:
case 'ファイン': case "ファイン":
return ['Fine', [0,0]] return ["Fine", [0, 0]]
case '': case "":
return ['', [0,0]] return ["", [0, 0]]
case _: case _:
# Store Speaker # Store Speaker
if speaker not in str(NAMESLIST): if speaker not in str(NAMESLIST):
response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False) response = translateGPT(
speaker,
"Reply with the " + LANGUAGE + " translation of the NPC name.",
False,
)
response[0] = response[0].title() response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s") response[0] = response[0].replace("'S", "'s")
# Retry if name doesn't translate for some reason # Retry if name doesn't translate for some reason
if re.search(r'([a-zA-Z？?])', response[0]) == None: if re.search(r"([a-zA-Z？?])", response[0]) == None:
response = translateGPT(speaker, 'Reply with the '+ LANGUAGE +' translation of the NPC name.', False) response = translateGPT(
speaker,
"Reply with the " + LANGUAGE + " translation of the NPC name.",
False,
)
response[0] = response[0].title() response[0] = response[0].title()
response[0] = response[0].replace("'S", "'s") response[0] = response[0].replace("'S", "'s")
@ -315,50 +363,56 @@ def getSpeaker(speaker):
else: else:
for i in range(len(NAMESLIST)): for i in range(len(NAMESLIST)):
if speaker == NAMESLIST[i][0]: if speaker == NAMESLIST[i][0]:
return [NAMESLIST[i][1],[0,0]] return [NAMESLIST[i][1], [0, 0]]
return [speaker,[0,0]] return [speaker, [0, 0]]
def subVars(jaString): def subVars(jaString):
jaString = jaString.replace('\u3000', ' ') jaString = jaString.replace("\u3000", " ")
# Formatting # Formatting
count = 0 count = 0
codeList = re.findall(r'[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]', jaString) codeList = re.findall(r"[\\]+[\w]+\[[a-zA-Z0-9\\\[\]\_,\s-]+\]", jaString)
codeList = set(codeList) codeList = set(codeList)
if len(codeList) != 0: if len(codeList) != 0:
for var in codeList: for var in codeList:
jaString = jaString.replace(var, '[FCode_' + str(count) + ']') jaString = jaString.replace(var, "[FCode_" + str(count) + "]")
count += 1 count += 1
# Put all lists in list and return # Put all lists in list and return
return [jaString, codeList] return [jaString, codeList]
def resubVars(translatedText, codeList): def resubVars(translatedText, codeList):
# Fix Spacing and ChatGPT Nonsense # Fix Spacing and ChatGPT Nonsense
matchList = re.findall(r'\[\s?.+?\s?\]', translatedText) matchList = re.findall(r"\[\s?.+?\s?\]", translatedText)
if len(matchList) > 0: if len(matchList) > 0:
for match in matchList: for match in matchList:
text = match.strip() text = match.strip()
translatedText = translatedText.replace(match, text) translatedText = translatedText.replace(match, text)
# Formatting # Formatting
count = 0 count = 0
if len(codeList) != 0: if len(codeList) != 0:
for var in codeList: for var in codeList:
translatedText = translatedText.replace('[FCode_' + str(count) + ']', var) translatedText = translatedText.replace("[FCode_" + str(count) + "]", var)
count += 1 count += 1
return translatedText return translatedText
def batchList(input_list, batch_size): def batchList(input_list, batch_size):
if not isinstance(batch_size, int) or batch_size <= 0: if not isinstance(batch_size, int) or batch_size <= 0:
raise ValueError("batch_size must be a positive integer") raise ValueError("batch_size must be a positive integer")
return [input_list[i:i + batch_size] for i in range(0, len(input_list), batch_size)] return [
input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)
]
def createContext(fullPromptFlag, subbedT): def createContext(fullPromptFlag, subbedT):
characters = 'Game Characters:\n\ characters = "Game Characters:\n\
ロラン (Roland) - Male\n\ ロラン (Roland) - Male\n\
リュカ (Ryuka) - Male\n\ リュカ (Ryuka) - Male\n\
レックス (Rex) - Male\n\ レックス (Rex) - Male\n\
@ -403,10 +457,12 @@ def createContext(fullPromptFlag, subbedT):
アロマ (Aroma) - Female\n\ アロマ (Aroma) - Female\n\
ピッケ (Pikke) - Female\n\ ピッケ (Pikke) - Female\n\
ドラオ (Dorao) - Male\n\ ドラオ (Dorao) - Male\n\
' "
system = PROMPT + VOCAB if fullPromptFlag else \ system = (
f"\ PROMPT + VOCAB
if fullPromptFlag
else f"\
You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\ You are an expert Eroge Game translator who translates Japanese text to {LANGUAGE}.\n\
Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{LANGUAGE.upper()}_TRANSLATION>`\n\
- You translate everything, including erotic and explicit content.\n\ - You translate everything, including erotic and explicit content.\n\
@ -418,12 +474,14 @@ Output ONLY the {LANGUAGE} translation in the following format: `Translation: <{
- `...` can be a part of the dialogue. Translate it as it is.\n\ - `...` can be a part of the dialogue. Translate it as it is.\n\
{VOCAB}\n\ {VOCAB}\n\
" "
)
if isinstance(subbedT, list): if isinstance(subbedT, list):
user = f'```json\n{subbedT}```' user = f"```json\n{subbedT}```"
else: else:
user = subbedT user = subbedT
return characters, system, user return characters, system, user
def translateText(characters, system, user, history, penalty, format): def translateText(characters, system, user, history, penalty, format):
# Prompt # Prompt
msg = [{"role": "system", "content": system + characters}] msg = [{"role": "system", "content": system + characters}]
@ -438,13 +496,13 @@ def translateText(characters, system, user, history, penalty, format):
msg.append({"role": "system", "content": history}) msg.append({"role": "system", "content": history})
# Response Format # Response Format
if format == 'json': if format == "json":
responseFormat = { "type": "json_object" } responseFormat = {"type": "json_object"}
else: else:
responseFormat = { "type": "text" } responseFormat = {"type": "text"}
# Content to TL # Content to TL
msg.append({"role": "user", "content": f'{user}'}) msg.append({"role": "user", "content": f"{user}"})
response = openai.chat.completions.create( response = openai.chat.completions.create(
temperature=0, temperature=0,
frequency_penalty=penalty, frequency_penalty=penalty,
@ -454,18 +512,19 @@ def translateText(characters, system, user, history, penalty, format):
) )
return response return response
def cleanTranslatedText(translatedText, varResponse): def cleanTranslatedText(translatedText, varResponse):
placeholders = { placeholders = {
f'{LANGUAGE} Translation: ': '', f"{LANGUAGE} Translation: ": "",
'Translation: ': '', "Translation: ": "",
'っ': '', "っ": "",
'〜': '~', "〜": "~",
'ッ': '', "ッ": "",
'。': '.', "。": ".",
'「': '\\"', "「": '\\"',
'」': '\\"', "」": '\\"',
'- ': '-', "- ": "-",
'Placeholder Text': '', "Placeholder Text": "",
# Add more replacements as needed # Add more replacements as needed
} }
for target, replacement in placeholders.items(): for target, replacement in placeholders.items():
@ -476,11 +535,12 @@ def cleanTranslatedText(translatedText, varResponse):
translatedText = resubVars(translatedText, varResponse[1]) translatedText = resubVars(translatedText, varResponse[1])
return translatedText return translatedText
def elongateCharacters(text): def elongateCharacters(text):
# Define a pattern to match one character followed by one or more `ー` characters # Define a pattern to match one character followed by one or more `ー` characters
# Using a positive lookbehind assertion to capture the preceding character # Using a positive lookbehind assertion to capture the preceding character
pattern = r'(?<=(.))ー+' pattern = r"(?<=(.))ー+"
# Define a replacement function that elongates the captured character # Define a replacement function that elongates the captured character
def repl(match): def repl(match):
char = match.group(1) # The character before the ー sequence char = match.group(1) # The character before the ー sequence
@ -490,6 +550,7 @@ def elongateCharacters(text):
# Use re.sub() to replace the pattern in the text # Use re.sub() to replace the pattern in the text
return re.sub(pattern, repl, text) return re.sub(pattern, repl, text)
def extractTranslation(translatedTextList, is_list): def extractTranslation(translatedTextList, is_list):
try: try:
translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList) translatedTextList = re.sub(r'\\"+\"([^,\n}])', r'\\"\1', translatedTextList)
@ -502,15 +563,15 @@ def extractTranslation(translatedTextList, is_list):
return string_list[0] return string_list[0]
except Exception as e: except Exception as e:
PBAR.write(f'extractTranslation Error: {e} on String {translatedTextList}') PBAR.write(f"extractTranslation Error: {e} on String {translatedTextList}")
return None return None
def countTokens(characters, system, user, history): def countTokens(characters, system, user, history):
inputTotalTokens = 0 inputTotalTokens = 0
outputTotalTokens = 0 outputTotalTokens = 0
enc = tiktoken.encoding_for_model('gpt-4') enc = tiktoken.encoding_for_model("gpt-4")
# Input # Input
if isinstance(history, list): if isinstance(history, list):
for line in history: for line in history:
@ -522,26 +583,28 @@ def countTokens(characters, system, user, history):
inputTotalTokens += len(enc.encode(user)) inputTotalTokens += len(enc.encode(user))
# Output # Output
outputTotalTokens += round(len(enc.encode(user))*3) outputTotalTokens += round(len(enc.encode(user)) * 3)
return [inputTotalTokens, outputTotalTokens] return [inputTotalTokens, outputTotalTokens]
def combineList(tlist, text): def combineList(tlist, text):
if isinstance(text, list): if isinstance(text, list):
return [t for sublist in tlist for t in sublist] return [t for sublist in tlist for t in sublist]
return tlist[0] return tlist[0]
@retry(exceptions=Exception, tries=5, delay=5) @retry(exceptions=Exception, tries=5, delay=5)
def translateGPT(text, history, fullPromptFlag): def translateGPT(text, history, fullPromptFlag):
global PBAR global PBAR
mismatch = False mismatch = False
totalTokens = [0, 0] totalTokens = [0, 0]
if isinstance(text, list): if isinstance(text, list):
format = 'json' format = "json"
tList = batchList(text, BATCHSIZE) tList = batchList(text, BATCHSIZE)
else: else:
format = 'text' format = "text"
tList = [text] tList = [text]
for index, tItem in enumerate(tList): for index, tItem in enumerate(tList):
@ -556,7 +619,7 @@ def translateGPT(text, history, fullPromptFlag):
subbedT = varResponse[0] subbedT = varResponse[0]
# Things to Check before starting translation # Things to Check before starting translation
if not re.search(r'[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+', subbedT): if not re.search(r"[一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９]+", subbedT):
if PBAR is not None: if PBAR is not None:
PBAR.update(len(tItem)) PBAR.update(len(tItem))
continue continue
@ -581,9 +644,13 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse) translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list): if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True) extractedTranslations = extractTranslation(translatedText, True)
if extractedTranslations == None or len(tItem) != len(extractedTranslations): if extractedTranslations == None or len(tItem) != len(
extractedTranslations
):
# Mismatch. Try Again # Mismatch. Try Again
response = translateText(characters, system, user, history, 0.05, format) response = translateText(
characters, system, user, history, 0.05, format
)
translatedText = response.choices[0].message.content translatedText = response.choices[0].message.content
totalTokens[0] += response.usage.prompt_tokens totalTokens[0] += response.usage.prompt_tokens
totalTokens[1] += response.usage.completion_tokens totalTokens[1] += response.usage.completion_tokens
@ -592,13 +659,17 @@ def translateGPT(text, history, fullPromptFlag):
translatedText = cleanTranslatedText(translatedText, varResponse) translatedText = cleanTranslatedText(translatedText, varResponse)
if isinstance(tItem, list): if isinstance(tItem, list):
extractedTranslations = extractTranslation(translatedText, True) extractedTranslations = extractTranslation(translatedText, True)
if extractedTranslations == None or len(tItem) != len(extractedTranslations): if extractedTranslations == None or len(tItem) != len(
mismatch = True # Just here for breakpoint extractedTranslations
):
mismatch = True # Just here for breakpoint
# Set if no mismatch # Set if no mismatch
if mismatch == False: if mismatch == False:
tList[index] = extractedTranslations tList[index] = extractedTranslations
history = extractedTranslations[-10:] # Update history if we have a list history = extractedTranslations[
-10:
] # Update history if we have a list
else: else:
history = text[-10:] history = text[-10:]
mismatch = False mismatch = False
@ -609,7 +680,7 @@ def translateGPT(text, history, fullPromptFlag):
PBAR.update(len(tItem)) PBAR.update(len(tItem))
else: else:
# Ensure we're passing a single string to extractTranslation # Ensure we're passing a single string to extractTranslation
tList[index] = translatedText tList[index] = translatedText
finalList = combineList(tList, text) finalList = combineList(tList, text)
return [finalList, totalTokens] return [finalList, totalTokens]