remove threads in modules and implement save progress for tls

This commit is contained in:
dazedanon 2025-08-24 15:07:07 -05:00
parent 217c19de5d
commit f535e3a4e4
14 changed files with 412 additions and 63 deletions

View file

@ -15,6 +15,7 @@ from dotenv import load_dotenv
from retry import retry
from tqdm import tqdm
from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
import tempfile
# Open AI
load_dotenv()
@ -206,7 +207,7 @@ def parseCSV(readFile, writeFile, filename):
data.append(row)
try:
response = translateCSV(data, pbar, writer, filename, None, format)
response = translateCSV(data, pbar, writeFile, writer, filename, None, format)
totalTokens[0] = response[0]
totalTokens[1] = response[1]
except Exception:
@ -214,7 +215,24 @@ def parseCSV(readFile, writeFile, filename):
return [data, totalTokens, None]
def translateCSV(data, pbar, writer, filename, translatedList, format):
def flush_progress_csv(writeFile, writer, rows):
"""Flush current CSV progress to the already-open output file (Windows-safe)."""
try:
if ESTIMATE or writeFile is None:
return
with LOCK:
writeFile.seek(0)
# Recreate writer at current position to avoid state issues
tmp_writer = csv.writer(writeFile, delimiter=",")
tmp_writer.writerows(rows)
writeFile.truncate()
writeFile.flush()
os.fsync(writeFile.fileno())
except Exception:
traceback.print_exc()
def translateCSV(data, pbar, writeFile, writer, filename, translatedList, format):
global LOCK, ESTIMATE, PBAR
PBAR = pbar
translatedText = ""
@ -254,6 +272,7 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
# Set Data
data[i][1] = translatedText
flush_progress_csv(writeFile, writer, data)
# Iterate
i += 1
@ -286,6 +305,7 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
# Set Data
data[i][targetColumn] = translatedText
flush_progress_csv(writeFile, writer, data)
# Iterate
i += 1
@ -369,6 +389,7 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
data[i][j + 1] = data[i][j + 1].replace(ojaString, f"{translatedText}")
else:
data[i][j] = data[i][j].replace(ojaString, f"{translatedText}")
flush_progress_csv(writeFile, writer, data)
# Iterate
i += 1
@ -429,6 +450,7 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
i += 1
else:
data[i][textColumn] = translatedText
flush_progress_csv(writeFile, writer, data)
# Iterate
i += 1
@ -460,6 +482,7 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
if not ESTIMATE:
for row in data:
writer.writerow(row)
flush_progress_csv(writeFile, writer, data)
except Exception:
traceback.print_exc()
@ -469,6 +492,7 @@ def translateCSV(data, pbar, writer, filename, translatedList, format):
if not ESTIMATE:
for row in data:
writer.writerow(row)
flush_progress_csv(writeFile, writer, data)
return totalTokens
return totalTokens

View file

@ -14,6 +14,7 @@ from dotenv import load_dotenv
from retry import retry
from tqdm import tqdm
from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost
import tempfile
# Open AI
load_dotenv()
@ -75,8 +76,9 @@ TRANSLATION_CONFIG = TranslationConfig(
LEAVE = False
def handleJSON(filename, estimate):
global ESTIMATE, totalTokens
global ESTIMATE, totalTokens, FILENAME
ESTIMATE = estimate
FILENAME = filename
if estimate:
start = time.time()
@ -160,7 +162,7 @@ def parseJSON(data, filename):
pbar.desc = filename
PBAR = pbar
try:
result = translateJSON(data, [])
result = translateJSON(data, filename, [])
totalTokens[0] += result[0]
totalTokens[1] += result[1]
except Exception as e:
@ -169,7 +171,30 @@ def parseJSON(data, filename):
return [data, totalTokens, None]
def translateJSON(data, translatedList):
def save_progress_json(data, filename):
"""Atomically save current JSON translation progress."""
try:
if ESTIMATE:
return
os.makedirs("translated", exist_ok=True)
tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
try:
with os.fdopen(tmp_fd, "w", encoding="utf-8", newline="\n") as tmp_file:
json.dump(data, tmp_file, ensure_ascii=False, indent=4)
os.replace(tmp_path, os.path.join("translated", filename))
finally:
# In case of exception before replace
if os.path.exists(tmp_path):
try:
os.remove(tmp_path)
except OSError:
pass
except Exception:
# Best-effort; don't crash the run on save failures
traceback.print_exc()
def translateJSON(data, filename, translatedList):
global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH
if translatedList:
stringList = translatedList[0]
@ -266,7 +291,10 @@ def translateJSON(data, translatedList):
if "" in data[i][messageKey] and "" not in translatedText:
data[i][messageKey] = data[i][messageKey].replace(originalString, f"{translatedText}")
else:
data[i][messageKey] = data[i][messageKey].replace(originalString, f"{translatedText}")
data[i][messageKey] = data[i][messageKey].replace(originalString, f"{translatedText}")
# Save progress after each message replacement
save_progress_json(data, filename)
# Next Value
i += 1
@ -290,7 +318,7 @@ def translateJSON(data, translatedList):
MISMATCH.append(FILENAME)
# Set Strings
translateJSON(data, [stringListTL])
translateJSON(data, filename, [stringListTL])
return tokens
# Save some money and enter the character before translation

View file

@ -14,6 +14,7 @@ from dotenv import load_dotenv
from retry import retry
from tqdm import tqdm
from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost
import tempfile
# Open AI
load_dotenv()
@ -181,6 +182,27 @@ def parseKiriKiri(readFile, filename):
return [data, totalTokens, None]
def save_progress_lines(lines, filename, encoding="cp932"):
"""Atomically save current line-based translation progress."""
try:
if ESTIMATE:
return
os.makedirs("translated", exist_ok=True)
tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
try:
with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
tmp_file.writelines(lines)
os.replace(tmp_path, os.path.join("translated", filename))
finally:
if os.path.exists(tmp_path):
try:
os.remove(tmp_path)
except OSError:
pass
except Exception:
traceback.print_exc()
def translateKiriKiri(data, pbar, filename, jobList):
# Check Job Data
if len(jobList) > 0:
@ -213,6 +235,7 @@ def translateKiriKiri(data, pbar, filename, jobList):
tokens[0] += response[1][0]
tokens[1] += response[1][1]
data[i] = data[i].replace(speakerJA, speaker)
save_progress_lines(data, filename)
i += 1
# Choices
@ -234,6 +257,7 @@ def translateKiriKiri(data, pbar, filename, jobList):
data[i] = data[i].replace("'", '"')
translatedText = translatedText.replace('"', "'")
data[i] = data[i].replace(jaString, translatedText)
save_progress_lines(data, filename)
# Dialogue
match = re.search(dialogueRegex, data[i])
@ -277,6 +301,7 @@ def translateKiriKiri(data, pbar, filename, jobList):
data[i] = data[i].replace("'", '"')
translatedText = translatedText.replace('"', "'")
data[i] = data[i].replace(jaString, translatedText)
save_progress_lines(data, filename)
# Next Line
i += 1

View file

@ -14,6 +14,7 @@ from dotenv import load_dotenv
from retry import retry
from tqdm import tqdm
from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
import tempfile
# Open AI
load_dotenv()
@ -168,6 +169,27 @@ def parseJSON(data, filename):
return [data, totalTokens, None]
def save_progress_json(data, filename):
"""Atomically save current JSON translation progress."""
try:
if ESTIMATE:
return
os.makedirs("translated", exist_ok=True)
tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
try:
with os.fdopen(tmp_fd, "w", encoding="utf-8", newline="\n") as tmp_file:
json.dump(data, tmp_file, ensure_ascii=False, indent=4)
os.replace(tmp_path, os.path.join("translated", filename))
finally:
if os.path.exists(tmp_path):
try:
os.remove(tmp_path)
except OSError:
pass
except Exception:
traceback.print_exc()
def translateJSON(data, pbar):
global PBAR
PBAR = pbar
@ -190,6 +212,7 @@ def translateJSON(data, pbar):
tokens[0] += response[1][0]
tokens[1] += response[1][1]
item["name"] = speaker
save_progress_json(data, FILENAME or "output.json")
else:
speaker = "None"
@ -259,6 +282,7 @@ def translateJSON(data, pbar):
# Set Text
item[text] = translatedText
save_progress_json(data, FILENAME or "output.json")
translatedBatch.pop(0)
speaker = ""
currentGroup = []
@ -295,6 +319,8 @@ def translateJSON(data, pbar):
batch.clear()
currentGroup = []
# After applying a batch, persist progress
save_progress_json(data, FILENAME or "output.json")
return tokens
# Save some money and enter the character before translation

View file

@ -14,6 +14,7 @@ from dotenv import load_dotenv
from retry import retry
from tqdm import tqdm
from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
import tempfile
# Open AI
load_dotenv()
@ -185,6 +186,27 @@ def parseOnscripter(readFile, filename):
return [data, totalTokens, None]
def save_progress_lines(lines, filename, encoding="cp932"):
"""Atomically save current line-based translation progress."""
try:
if ESTIMATE:
return
os.makedirs("translated", exist_ok=True)
tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
try:
with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
tmp_file.writelines(lines)
os.replace(tmp_path, os.path.join("translated", filename))
finally:
if os.path.exists(tmp_path):
try:
os.remove(tmp_path)
except OSError:
pass
except Exception:
traceback.print_exc()
def translateOnscripter(data, pbar, filename, translatedList):
stringList = []
currentGroup = []
@ -272,6 +294,7 @@ def translateOnscripter(data, pbar, filename, translatedList):
# Set Data
data[i] = data[i].replace(originalString, f"{translatedText}")
save_progress_lines(data, filename)
i += 1
# Choices
@ -295,6 +318,7 @@ def translateOnscripter(data, pbar, filename, translatedList):
# Set
data[i] = data[i].replace(choiceList[j], translatedText)
save_progress_lines(data, filename)
i += 1
# Nothing relevant. Skip Line.

View file

@ -14,6 +14,7 @@ from dotenv import load_dotenv
from retry import retry
from tqdm import tqdm
from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
import tempfile
# Open AI
load_dotenv()
@ -168,7 +169,7 @@ def parseRegex(readFile, filename):
PBAR = pbar
try:
result = translateRegex(data, [])
result = translateRegex(data, filename, [])
totalTokens[0] += result[0]
totalTokens[1] += result[1]
except Exception as e:
@ -177,7 +178,28 @@ def parseRegex(readFile, filename):
return [data, totalTokens, None]
def translateRegex(data, translatedList):
def save_progress_lines(lines, filename, encoding="utf-8-sig"):
"""Atomically save current line-based translation progress."""
try:
if ESTIMATE:
return
os.makedirs("translated", exist_ok=True)
tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
try:
with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
tmp_file.writelines(lines)
os.replace(tmp_path, os.path.join("translated", filename))
finally:
if os.path.exists(tmp_path):
try:
os.remove(tmp_path)
except OSError:
pass
except Exception:
traceback.print_exc()
def translateRegex(data, filename, translatedList):
if translatedList:
stringList = translatedList[0]
choiceList = translatedList[1]
@ -213,6 +235,7 @@ def translateRegex(data, translatedList):
if not translatedList:
title = re.sub(r"(?<!\\)'", r"\\'", title)
data[i] = data[i].replace(match.group(1), title)
save_progress_lines(data, filename)
# Speaker
match = re.search(lineRegexSpeaker, data[i])
@ -227,6 +250,7 @@ def translateRegex(data, translatedList):
tokens[1] += response[1][1]
if translatedList:
data[i] = data[i].replace(match.group(1), speaker)
save_progress_lines(data, filename)
i += 3
else:
speaker = None
@ -287,7 +311,8 @@ def translateRegex(data, translatedList):
if "" in data[i-1] and "" not in translatedText:
data[i] = data[i].replace(originalString, f"{translatedText}")
else:
data[i] = data[i].replace(originalString, f"{translatedText}")
data[i] = data[i].replace(originalString, f"{translatedText}")
save_progress_lines(data, filename)
# Choices
match = re.search(choiceRegex, data[i])
@ -311,6 +336,7 @@ def translateRegex(data, translatedList):
# Set
data[i] = data[i].replace(match.group(1), translatedText)
save_progress_lines(data, filename)
i += 1
else:
@ -350,7 +376,7 @@ def translateRegex(data, translatedList):
MISMATCH.append(FILENAME)
# Set Strings
translateRegex(data, [stringListTL, choiceListTL])
translateRegex(data, filename, [stringListTL, choiceListTL])
return tokens
# Save some money and enter the character before translation

View file

@ -14,6 +14,7 @@ from dotenv import load_dotenv
from retry import retry
from tqdm import tqdm
from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost
import tempfile
# Open AI
load_dotenv()
@ -172,7 +173,7 @@ def parseRenpy(readFile, filename):
PBAR = pbar
try:
result = translateRenpy(data, [])
result = translateRenpy(data, filename, [])
totalTokens[0] += result[0]
totalTokens[1] += result[1]
except Exception as e:
@ -181,7 +182,28 @@ def parseRenpy(readFile, filename):
return [data, totalTokens, None]
def translateRenpy(data, translatedList):
def save_progress_lines(lines, filename, encoding="utf-8"):
"""Atomically save current line-based translation progress."""
try:
if ESTIMATE:
return
os.makedirs("translated", exist_ok=True)
tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
try:
with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
tmp_file.writelines(lines)
os.replace(tmp_path, os.path.join("translated", filename))
finally:
if os.path.exists(tmp_path):
try:
os.remove(tmp_path)
except OSError:
pass
except Exception:
traceback.print_exc()
def translateRenpy(data, filename, translatedList):
stringList = []
currentGroup = []
tokens = [0, 0]
@ -250,6 +272,7 @@ def translateRenpy(data, translatedList):
# Set Data
data[i] = data[i].replace(originalString, translatedText)
save_progress_lines(data, filename)
i += 1
else:
i += 1
@ -268,7 +291,7 @@ def translateRenpy(data, translatedList):
# Set Strings
if len(stringList) == len(translatedList):
translateRenpy(data, translatedList)
translateRenpy(data, filename, translatedList)
# Mismatch
else:

View file

@ -9,7 +9,7 @@ import traceback
import tiktoken
import openai
import copy
from concurrent.futures import ThreadPoolExecutor, as_completed
# Removed concurrent.futures usage for simplicity; running synchronously
from pathlib import Path
from colorama import Fore
from dotenv import load_dotenv
@ -242,6 +242,24 @@ def getResultString(translatedData, translationTime, filename):
return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
def save_progress_yaml(data, filename):
"""Atomically write current YAML data to translated/filename; skip in estimate mode."""
try:
if ESTIMATE:
return
os.makedirs("translated", exist_ok=True)
tmp_path = os.path.join("translated", f"{filename}.tmp")
final_path = os.path.join("translated", filename)
yaml = YAML(pure=True)
yaml.width = 4096
yaml.default_style = "'"
with open(tmp_path, "w", encoding="utf-8", newline="\n") as outFile:
yaml.dump(data, outFile)
os.replace(tmp_path, final_path)
except Exception:
traceback.print_exc()
def parseMap(data, filename):
totalTokens = [0, 0]
totalLines = 0
@ -259,26 +277,27 @@ def parseMap(data, filename):
totalTokens[1] += response[1][1]
data["display_name"] = response[0].replace('"', "")
# Thread for each page in file
# Process each page synchronously and persist after each
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
pbar.desc = filename
with ThreadPoolExecutor(max_workers=THREADS) as executor:
for key in events:
if key is not None:
# This translates ID of events. (May break the game)
if "<namepop" in events[key]["name"]:
response = translateNoteOmitSpace(events[key], r"<namepop\s(.*?)\s?\d?>.*")
totalTokens[0] += response[0]
totalTokens[1] += response[1]
futures = [executor.submit(searchCodes, page, pbar, [], filename) for page in events[key]["pages"] if page is not None]
for future in as_completed(futures):
for key in events:
if key is not None:
# This translates ID of events. (May break the game)
if "<namepop" in events[key]["name"]:
response = translateNoteOmitSpace(events[key], r"<namepop\s(.*?)\s?\d?>.*")
totalTokens[0] += response[0]
totalTokens[1] += response[1]
for page in events[key]["pages"]:
if page is not None:
try:
totalTokensFuture = future.result()
totalTokens[0] += totalTokensFuture[0]
totalTokens[1] += totalTokensFuture[1]
tt = searchCodes(page, pbar, [], filename)
totalTokens[0] += tt[0]
totalTokens[1] += tt[1]
except Exception as e:
traceback.print_exc()
return [data, totalTokens, e]
finally:
save_progress_yaml(data, filename)
return [data, totalTokens, None]
@ -356,16 +375,17 @@ def parseCommonEvents(data, filename):
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
pbar.desc = filename
with ThreadPoolExecutor(max_workers=THREADS) as executor:
futures = [executor.submit(searchCodes, page, pbar, [], filename) for page in data if page is not None]
for future in as_completed(futures):
for page in data:
if page is not None:
try:
totalTokensFuture = future.result()
totalTokens[0] += totalTokensFuture[0]
totalTokens[1] += totalTokensFuture[1]
tt = searchCodes(page, pbar, [], filename)
totalTokens[0] += tt[0]
totalTokens[1] += tt[1]
except Exception as e:
traceback.print_exc()
return [data, totalTokens, e]
finally:
save_progress_yaml(data, filename)
return [data, totalTokens, None]
@ -384,16 +404,17 @@ def parseTroops(data, filename):
pbar.desc = filename
for troop in data:
if troop is not None:
with ThreadPoolExecutor(max_workers=THREADS) as executor:
futures = [executor.submit(searchCodes, page, pbar, [], filename) for page in troop["pages"] if page is not None]
for future in as_completed(futures):
for page in troop["pages"]:
if page is not None:
try:
totalTokensFuture = future.result()
totalTokens[0] += totalTokensFuture[0]
totalTokens[1] += totalTokensFuture[1]
tt = searchCodes(page, pbar, [], filename)
totalTokens[0] += tt[0]
totalTokens[1] += tt[1]
except Exception as e:
traceback.print_exc()
return [data, totalTokens, e]
finally:
save_progress_yaml(data, filename)
return [data, totalTokens, None]
@ -411,6 +432,8 @@ def parseNames(data, filename, context):
except Exception as e:
traceback.print_exc()
return [data, totalTokens, e]
finally:
save_progress_yaml(data, filename)
return [data, totalTokens, None]
@ -430,6 +453,8 @@ def parseSS(data, filename):
except Exception as e:
traceback.print_exc()
return [data, totalTokens, e]
finally:
save_progress_yaml(data, filename)
return [data, totalTokens, None]
@ -456,6 +481,8 @@ def parseSystem(data, filename):
except Exception as e:
traceback.print_exc()
return [data, totalTokens, e]
finally:
save_progress_yaml(data, filename)
return [data, totalTokens, None]
@ -470,16 +497,17 @@ def parseScenario(data, filename):
with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar:
pbar.desc = filename
with ThreadPoolExecutor(max_workers=THREADS) as executor:
futures = [executor.submit(searchCodes, page[1], pbar, [], filename) for page in data.items() if page[1] is not None]
for future in as_completed(futures):
for page in data.items():
if page[1] is not None:
try:
totalTokensFuture = future.result()
totalTokens[0] += totalTokensFuture[0]
totalTokens[1] += totalTokensFuture[1]
tt = searchCodes(page[1], pbar, [], filename)
totalTokens[0] += tt[0]
totalTokens[1] += tt[1]
except Exception as e:
traceback.print_exc()
return [data, totalTokens, e]
finally:
save_progress_yaml(data, filename)
return [data, totalTokens, None]

View file

@ -14,6 +14,7 @@ from dotenv import load_dotenv
from retry import retry
from tqdm import tqdm
from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
import tempfile
# Open AI
load_dotenv()
@ -176,6 +177,27 @@ def parsePlugin(readFile, filename):
return [data, totalTokens, None]
def save_progress_lines(lines, filename, encoding="utf_8"):
"""Atomically save current line-based translation progress."""
try:
if ESTIMATE:
return
os.makedirs("translated", exist_ok=True)
tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
try:
with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
tmp_file.writelines(lines)
os.replace(tmp_path, os.path.join("translated", filename))
finally:
if os.path.exists(tmp_path):
try:
os.remove(tmp_path)
except OSError:
pass
except Exception:
traceback.print_exc()
def translatePlugin(data, pbar, filename, translatedList):
if len(translatedList) > 0:
questList = translatedList[0]
@ -259,6 +281,7 @@ def translatePlugin(data, pbar, filename, translatedList):
with open("translations.txt", "a+", encoding="utf-8") as tlFile:
tlFile.write(f"{originalString} ({translatedText})\n")
data[i] = data[i].replace(originalString, translatedText)
save_progress_lines(data, filename)
# Quest Name
regex = r'[\\]+"QuestName[\\]+":[\\]+"(.*?)[\\]+"'
@ -295,6 +318,7 @@ def translatePlugin(data, pbar, filename, translatedList):
# Set Data
data[i] = data[i].replace(originalString, translatedText)
save_progress_lines(data, filename)
# Quest Client
regex = r'QuestClientName[\\]+":[\\]+"(.*?)[\\]+"'
@ -328,6 +352,7 @@ def translatePlugin(data, pbar, filename, translatedList):
# Set Data
data[i] = data[i].replace(originalString, translatedText)
save_progress_lines(data, filename)
# Quest Location
regex = r'QuestLocation[\\]+":[\\]+"(.*?)[\\]+"'
@ -361,6 +386,7 @@ def translatePlugin(data, pbar, filename, translatedList):
# Set Data
data[i] = data[i].replace(originalString, translatedText)
save_progress_lines(data, filename)
# Quest Target
regex = r'PlaceInformation[\\]+":[\\]+"(.*?)[\\]+"'
@ -394,6 +420,7 @@ def translatePlugin(data, pbar, filename, translatedList):
# Set Data
data[i] = data[i].replace(originalString, translatedText)
save_progress_lines(data, filename)
# Quest Summary
regex = r'[\\]+"QuestContent[\\]+":[\\]+"[\\]+"(.*?)[\\]+"[\\]+"'
@ -436,6 +463,7 @@ def translatePlugin(data, pbar, filename, translatedList):
# Set Data
data[i] = data[i].replace(originalString, translatedText)
save_progress_lines(data, filename)
# Quest Goal 1
regex = r'ObjectiveContent[\\]+":[\\]+"[\\]+"(.*?)[\\]+"'
@ -476,6 +504,7 @@ def translatePlugin(data, pbar, filename, translatedList):
# Set Data
data[i] = data[i].replace(originalString, translatedText)
save_progress_lines(data, filename)
# Next Line
i += 1

View file

@ -14,6 +14,7 @@ from dotenv import load_dotenv
from retry import retry
from tqdm import tqdm
from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost
import tempfile
# Open AI
load_dotenv()
@ -161,7 +162,7 @@ def parseText(readFile, filename):
PBAR = pbar
try:
result = translateTxt(data, [])
result = translateTxt(data, filename, [])
totalTokens[0] += result[0]
totalTokens[1] += result[1]
except Exception as e:
@ -170,7 +171,28 @@ def parseText(readFile, filename):
return [data, totalTokens, None]
def translateTxt(data, translatedList):
def save_progress_lines(lines, filename, encoding="utf-8"):
"""Atomically save current line-based translation progress."""
try:
if ESTIMATE:
return
os.makedirs("translated", exist_ok=True)
tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
try:
with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
tmp_file.writelines(lines)
os.replace(tmp_path, os.path.join("translated", filename))
finally:
if os.path.exists(tmp_path):
try:
os.remove(tmp_path)
except OSError:
pass
except Exception:
traceback.print_exc()
def translateTxt(data, filename, translatedList):
if translatedList:
stringList = translatedList[0]
choiceList = translatedList[1]
@ -224,6 +246,7 @@ def translateTxt(data, translatedList):
# Set Data
data[i] = f"{translatedText}\n"
save_progress_lines(data, filename)
i += 1
else:
@ -263,7 +286,7 @@ def translateTxt(data, translatedList):
MISMATCH.append(FILENAME)
# Set Strings
translateTxt(data, [stringListTL, choiceListTL])
translateTxt(data, filename, [stringListTL, choiceListTL])
return tokens
# Save some money and enter the character before translation

View file

@ -14,6 +14,7 @@ from dotenv import load_dotenv
from retry import retry
from tqdm import tqdm
from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
import tempfile
# Open AI
load_dotenv()
@ -161,7 +162,7 @@ def parseTyrano(readFile, filename):
PBAR = pbar
try:
result = translateTyrano(data, [])
result = translateTyrano(data, filename, [])
totalTokens[0] += result[0]
totalTokens[1] += result[1]
except Exception as e:
@ -170,7 +171,28 @@ def parseTyrano(readFile, filename):
return [data, totalTokens, None]
def translateTyrano(data, translatedList):
def save_progress_lines(lines, filename, encoding="utf-8"):
"""Atomically save current line-based translation progress."""
try:
if ESTIMATE:
return
os.makedirs("translated", exist_ok=True)
tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
try:
with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
tmp_file.writelines(lines)
os.replace(tmp_path, os.path.join("translated", filename))
finally:
if os.path.exists(tmp_path):
try:
os.remove(tmp_path)
except OSError:
pass
except Exception:
traceback.print_exc()
def translateTyrano(data, filename, translatedList):
if translatedList:
stringList = translatedList[0]
choiceList = translatedList[1]
@ -291,6 +313,8 @@ def translateTyrano(data, translatedList):
# Set Data
data[i] = data[i].replace(originalString, translatedText)
# Save progress after each line change
save_progress_lines(data, filename)
# Choices
match = re.search(choiceRegex, data[i])
@ -311,6 +335,7 @@ def translateTyrano(data, translatedList):
# Set
data[i] = data[i].replace(match.group(1), translatedText)
save_progress_lines(data, filename)
i += 1
else:
@ -350,7 +375,7 @@ def translateTyrano(data, translatedList):
MISMATCH.append(FILENAME)
# Set Strings
translateTyrano(data, [stringListTL, choiceListTL])
translateTyrano(data, filename, [stringListTL, choiceListTL])
return tokens
# Save some money and enter the character before translation

View file

@ -14,6 +14,7 @@ from dotenv import load_dotenv
from retry import retry
from tqdm import tqdm
from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost
import tempfile
# Open AI
load_dotenv()
@ -184,6 +185,27 @@ def parseUnity(readFile, filename):
return [data, totalTokens, None]
def save_progress_lines(lines, filename, encoding="utf-8"):
"""Atomically save current line-based translation progress."""
try:
if ESTIMATE:
return
os.makedirs("translated", exist_ok=True)
tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
try:
with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
tmp_file.writelines(lines)
os.replace(tmp_path, os.path.join("translated", filename))
finally:
if os.path.exists(tmp_path):
try:
os.remove(tmp_path)
except OSError:
pass
except Exception:
traceback.print_exc()
def translateUnity(data, pbar, filename, translatedList):
stringList = []
tokens = [0, 0]
@ -237,6 +259,7 @@ def translateUnity(data, pbar, filename, translatedList):
# Set Data
data[i] = f"{leftString}={translatedText}\n"
save_progress_lines(data, filename)
i += 1
# Nothing relevant. Skip Line.

View file

@ -8,7 +8,7 @@ import time
import traceback
import tiktoken
import openai
from concurrent.futures import ThreadPoolExecutor, as_completed
# Removed concurrent.futures usage for simplicity; running synchronously
from pathlib import Path
from colorama import Fore
from dotenv import load_dotenv
@ -190,6 +190,21 @@ def getResultString(translatedData, translationTime, filename):
return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET
def save_progress_json(data, filename):
"""Atomically write current JSON data to translated/filename; skip in estimate mode."""
try:
if ESTIMATE:
return
os.makedirs("translated", exist_ok=True)
tmp_path = os.path.join("translated", f"{filename}.tmp")
final_path = os.path.join("translated", filename)
with open(tmp_path, "w", encoding="utf-8", newline="\n") as outFile:
json.dump(data, outFile, ensure_ascii=False, indent=4)
os.replace(tmp_path, final_path)
except Exception:
traceback.print_exc()
def parseOther(data, filename):
totalTokens = [0, 0]
totalLines = 0
@ -206,6 +221,8 @@ def parseOther(data, filename):
totalTokens[1] += translationData[1]
except Exception as e:
return [data, totalTokens, e]
finally:
save_progress_json(data, filename)
return [data, totalTokens, None]
@ -225,6 +242,8 @@ def parseDB(data, filename):
totalTokens[1] += translationData[1]
except Exception as e:
return [data, totalTokens, e]
finally:
save_progress_json(data, filename)
return [data, totalTokens, None]
@ -240,21 +259,22 @@ def parseMap(data, filename):
for page in event["pages"]:
totalLines += len(page["list"])
# Thread for each page in file
# Process pages synchronously and persist after each
with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar:
pbar.desc = filename
pbar.total = totalLines
with ThreadPoolExecutor(max_workers=THREADS) as executor:
for event in events:
if event is not None:
futures = [executor.submit(searchCodes, page["list"], pbar, None, filename) for page in event["pages"] if page is not None]
for future in as_completed(futures):
for event in events:
if event is not None:
for page in event["pages"]:
if page is not None:
try:
totalTokensFuture = future.result()
totalTokens[0] += totalTokensFuture[0]
totalTokens[1] += totalTokensFuture[1]
tt = searchCodes(page["list"], pbar, None, filename)
totalTokens[0] += tt[0]
totalTokens[1] += tt[1]
except Exception as e:
return [data, totalTokens, e]
finally:
save_progress_json(data, filename)
return [data, totalTokens, None]

View file

@ -14,6 +14,7 @@ from dotenv import load_dotenv
from retry import retry
from tqdm import tqdm
from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost
import tempfile
# Open AI
load_dotenv()
@ -175,6 +176,27 @@ def parseWOLF(readFile, filename):
return [data, totalTokens, None]
def save_progress_lines(lines, filename, encoding="shift_jis"):
"""Atomically save current line-based translation progress."""
try:
if ESTIMATE:
return
os.makedirs("translated", exist_ok=True)
tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated")
try:
with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file:
tmp_file.writelines(lines)
os.replace(tmp_path, os.path.join("translated", filename))
finally:
if os.path.exists(tmp_path):
try:
os.remove(tmp_path)
except OSError:
pass
except Exception:
traceback.print_exc()
def translateWOLF(data, translatedList, pbar, filename):
stringList = []
currentGroup = []
@ -193,6 +215,7 @@ def translateWOLF(data, translatedList, pbar, filename):
tokens[0] += response[1][0]
tokens[1] += response[1][1]
data[i] = data[i].replace(matchList[0], f"{speaker}")
save_progress_lines(data, filename)
i += 1
else:
speaker = ""
@ -221,6 +244,7 @@ def translateWOLF(data, translatedList, pbar, filename):
data[i] = f"//{choiceListTL[0]}\n"
choiceListTL.pop(0)
i += 1
save_progress_lines(data, filename)
# Mismatch
else:
@ -278,6 +302,7 @@ def translateWOLF(data, translatedList, pbar, filename):
# Set Data
data.insert(i, f"{translatedText}\n")
save_progress_lines(data, filename)
i += 1
# Nothing relevant. Skip Line.