diff --git a/modules/csv.py b/modules/csv.py index af7406e..8a8a5c7 100644 --- a/modules/csv.py +++ b/modules/csv.py @@ -15,6 +15,7 @@ from dotenv import load_dotenv from retry import retry from tqdm import tqdm from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost +import tempfile # Open AI load_dotenv() @@ -206,7 +207,7 @@ def parseCSV(readFile, writeFile, filename): data.append(row) try: - response = translateCSV(data, pbar, writer, filename, None, format) + response = translateCSV(data, pbar, writeFile, writer, filename, None, format) totalTokens[0] = response[0] totalTokens[1] = response[1] except Exception: @@ -214,7 +215,24 @@ def parseCSV(readFile, writeFile, filename): return [data, totalTokens, None] -def translateCSV(data, pbar, writer, filename, translatedList, format): +def flush_progress_csv(writeFile, writer, rows): + """Flush current CSV progress to the already-open output file (Windows-safe).""" + try: + if ESTIMATE or writeFile is None: + return + with LOCK: + writeFile.seek(0) + # Recreate writer at current position to avoid state issues + tmp_writer = csv.writer(writeFile, delimiter=",") + tmp_writer.writerows(rows) + writeFile.truncate() + writeFile.flush() + os.fsync(writeFile.fileno()) + except Exception: + traceback.print_exc() + + +def translateCSV(data, pbar, writeFile, writer, filename, translatedList, format): global LOCK, ESTIMATE, PBAR PBAR = pbar translatedText = "" @@ -254,6 +272,7 @@ def translateCSV(data, pbar, writer, filename, translatedList, format): # Set Data data[i][1] = translatedText + flush_progress_csv(writeFile, writer, data) # Iterate i += 1 @@ -286,6 +305,7 @@ def translateCSV(data, pbar, writer, filename, translatedList, format): # Set Data data[i][targetColumn] = translatedText + flush_progress_csv(writeFile, writer, data) # Iterate i += 1 @@ -369,6 +389,7 @@ def translateCSV(data, pbar, writer, filename, translatedList, format): data[i][j + 1] = data[i][j + 1].replace(ojaString, f"{translatedText}") else: data[i][j] = data[i][j].replace(ojaString, f"{translatedText}") + flush_progress_csv(writeFile, writer, data) # Iterate i += 1 @@ -429,6 +450,7 @@ def translateCSV(data, pbar, writer, filename, translatedList, format): i += 1 else: data[i][textColumn] = translatedText + flush_progress_csv(writeFile, writer, data) # Iterate i += 1 @@ -460,6 +482,7 @@ def translateCSV(data, pbar, writer, filename, translatedList, format): if not ESTIMATE: for row in data: writer.writerow(row) + flush_progress_csv(writeFile, writer, data) except Exception: traceback.print_exc() @@ -469,6 +492,7 @@ def translateCSV(data, pbar, writer, filename, translatedList, format): if not ESTIMATE: for row in data: writer.writerow(row) + flush_progress_csv(writeFile, writer, data) return totalTokens return totalTokens diff --git a/modules/json.py b/modules/json.py index 2bef315..03fddab 100644 --- a/modules/json.py +++ b/modules/json.py @@ -14,6 +14,7 @@ from dotenv import load_dotenv from retry import retry from tqdm import tqdm from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost +import tempfile # Open AI load_dotenv() @@ -75,8 +76,9 @@ TRANSLATION_CONFIG = TranslationConfig( LEAVE = False def handleJSON(filename, estimate): - global ESTIMATE, totalTokens + global ESTIMATE, totalTokens, FILENAME ESTIMATE = estimate + FILENAME = filename if estimate: start = time.time() @@ -160,7 +162,7 @@ def parseJSON(data, filename): pbar.desc = filename PBAR = pbar try: - result = translateJSON(data, []) + result = translateJSON(data, filename, []) totalTokens[0] += result[0] totalTokens[1] += result[1] except Exception as e: @@ -169,7 +171,30 @@ def parseJSON(data, filename): return [data, totalTokens, None] -def translateJSON(data, translatedList): +def save_progress_json(data, filename): + """Atomically save current JSON translation progress.""" + try: + if ESTIMATE: + return + os.makedirs("translated", exist_ok=True) + tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") + try: + with os.fdopen(tmp_fd, "w", encoding="utf-8", newline="\n") as tmp_file: + json.dump(data, tmp_file, ensure_ascii=False, indent=4) + os.replace(tmp_path, os.path.join("translated", filename)) + finally: + # In case of exception before replace + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + except Exception: + # Best-effort; don't crash the run on save failures + traceback.print_exc() + + +def translateJSON(data, filename, translatedList): global LOCK, ESTIMATE, FILENAME, PBAR, MISMATCH if translatedList: stringList = translatedList[0] @@ -266,7 +291,10 @@ def translateJSON(data, translatedList): if "『" in data[i][messageKey] and "』" not in translatedText: data[i][messageKey] = data[i][messageKey].replace(originalString, f"『{translatedText}』") else: - data[i][messageKey] = data[i][messageKey].replace(originalString, f"{translatedText}") + data[i][messageKey] = data[i][messageKey].replace(originalString, f"{translatedText}") + + # Save progress after each message replacement + save_progress_json(data, filename) # Next Value i += 1 @@ -290,7 +318,7 @@ def translateJSON(data, translatedList): MISMATCH.append(FILENAME) # Set Strings - translateJSON(data, [stringListTL]) + translateJSON(data, filename, [stringListTL]) return tokens # Save some money and enter the character before translation diff --git a/modules/kirikiri.py b/modules/kirikiri.py index 00d57f2..f9bb038 100644 --- a/modules/kirikiri.py +++ b/modules/kirikiri.py @@ -14,6 +14,7 @@ from dotenv import load_dotenv from retry import retry from tqdm import tqdm from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost +import tempfile # Open AI load_dotenv() @@ -181,6 +182,27 @@ def parseKiriKiri(readFile, filename): return [data, totalTokens, None] +def save_progress_lines(lines, filename, encoding="cp932"): + """Atomically save current line-based translation progress.""" + try: + if ESTIMATE: + return + os.makedirs("translated", exist_ok=True) + tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") + try: + with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: + tmp_file.writelines(lines) + os.replace(tmp_path, os.path.join("translated", filename)) + finally: + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + except Exception: + traceback.print_exc() + + def translateKiriKiri(data, pbar, filename, jobList): # Check Job Data if len(jobList) > 0: @@ -213,6 +235,7 @@ def translateKiriKiri(data, pbar, filename, jobList): tokens[0] += response[1][0] tokens[1] += response[1][1] data[i] = data[i].replace(speakerJA, speaker) + save_progress_lines(data, filename) i += 1 # Choices @@ -234,6 +257,7 @@ def translateKiriKiri(data, pbar, filename, jobList): data[i] = data[i].replace("'", '"') translatedText = translatedText.replace('"', "'") data[i] = data[i].replace(jaString, translatedText) + save_progress_lines(data, filename) # Dialogue match = re.search(dialogueRegex, data[i]) @@ -277,6 +301,7 @@ def translateKiriKiri(data, pbar, filename, jobList): data[i] = data[i].replace("'", '"') translatedText = translatedText.replace('"', "'") data[i] = data[i].replace(jaString, translatedText) + save_progress_lines(data, filename) # Next Line i += 1 diff --git a/modules/lune.py b/modules/lune.py index 87ed4fa..c2dca54 100644 --- a/modules/lune.py +++ b/modules/lune.py @@ -14,6 +14,7 @@ from dotenv import load_dotenv from retry import retry from tqdm import tqdm from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost +import tempfile # Open AI load_dotenv() @@ -168,6 +169,27 @@ def parseJSON(data, filename): return [data, totalTokens, None] +def save_progress_json(data, filename): + """Atomically save current JSON translation progress.""" + try: + if ESTIMATE: + return + os.makedirs("translated", exist_ok=True) + tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") + try: + with os.fdopen(tmp_fd, "w", encoding="utf-8", newline="\n") as tmp_file: + json.dump(data, tmp_file, ensure_ascii=False, indent=4) + os.replace(tmp_path, os.path.join("translated", filename)) + finally: + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + except Exception: + traceback.print_exc() + + def translateJSON(data, pbar): global PBAR PBAR = pbar @@ -190,6 +212,7 @@ def translateJSON(data, pbar): tokens[0] += response[1][0] tokens[1] += response[1][1] item["name"] = speaker + save_progress_json(data, FILENAME or "output.json") else: speaker = "None" @@ -259,6 +282,7 @@ def translateJSON(data, pbar): # Set Text item[text] = translatedText + save_progress_json(data, FILENAME or "output.json") translatedBatch.pop(0) speaker = "" currentGroup = [] @@ -295,6 +319,8 @@ def translateJSON(data, pbar): batch.clear() currentGroup = [] + # After applying a batch, persist progress + save_progress_json(data, FILENAME or "output.json") return tokens # Save some money and enter the character before translation diff --git a/modules/nscript.py b/modules/nscript.py index 9f2afa6..81568b5 100644 --- a/modules/nscript.py +++ b/modules/nscript.py @@ -14,6 +14,7 @@ from dotenv import load_dotenv from retry import retry from tqdm import tqdm from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost +import tempfile # Open AI load_dotenv() @@ -185,6 +186,27 @@ def parseOnscripter(readFile, filename): return [data, totalTokens, None] +def save_progress_lines(lines, filename, encoding="cp932"): + """Atomically save current line-based translation progress.""" + try: + if ESTIMATE: + return + os.makedirs("translated", exist_ok=True) + tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") + try: + with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: + tmp_file.writelines(lines) + os.replace(tmp_path, os.path.join("translated", filename)) + finally: + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + except Exception: + traceback.print_exc() + + def translateOnscripter(data, pbar, filename, translatedList): stringList = [] currentGroup = [] @@ -272,6 +294,7 @@ def translateOnscripter(data, pbar, filename, translatedList): # Set Data data[i] = data[i].replace(originalString, f"{translatedText}") + save_progress_lines(data, filename) i += 1 # Choices @@ -295,6 +318,7 @@ def translateOnscripter(data, pbar, filename, translatedList): # Set data[i] = data[i].replace(choiceList[j], translatedText) + save_progress_lines(data, filename) i += 1 # Nothing relevant. Skip Line. diff --git a/modules/regex.py b/modules/regex.py index 4cdba16..a30ee9d 100644 --- a/modules/regex.py +++ b/modules/regex.py @@ -14,6 +14,7 @@ from dotenv import load_dotenv from retry import retry from tqdm import tqdm from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost +import tempfile # Open AI load_dotenv() @@ -168,7 +169,7 @@ def parseRegex(readFile, filename): PBAR = pbar try: - result = translateRegex(data, []) + result = translateRegex(data, filename, []) totalTokens[0] += result[0] totalTokens[1] += result[1] except Exception as e: @@ -177,7 +178,28 @@ def parseRegex(readFile, filename): return [data, totalTokens, None] -def translateRegex(data, translatedList): +def save_progress_lines(lines, filename, encoding="utf-8-sig"): + """Atomically save current line-based translation progress.""" + try: + if ESTIMATE: + return + os.makedirs("translated", exist_ok=True) + tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") + try: + with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: + tmp_file.writelines(lines) + os.replace(tmp_path, os.path.join("translated", filename)) + finally: + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + except Exception: + traceback.print_exc() + + +def translateRegex(data, filename, translatedList): if translatedList: stringList = translatedList[0] choiceList = translatedList[1] @@ -213,6 +235,7 @@ def translateRegex(data, translatedList): if not translatedList: title = re.sub(r"(?.*") - totalTokens[0] += response[0] - totalTokens[1] += response[1] - futures = [executor.submit(searchCodes, page, pbar, [], filename) for page in events[key]["pages"] if page is not None] - for future in as_completed(futures): + for key in events: + if key is not None: + # This translates ID of events. (May break the game) + if ".*") + totalTokens[0] += response[0] + totalTokens[1] += response[1] + for page in events[key]["pages"]: + if page is not None: try: - totalTokensFuture = future.result() - totalTokens[0] += totalTokensFuture[0] - totalTokens[1] += totalTokensFuture[1] + tt = searchCodes(page, pbar, [], filename) + totalTokens[0] += tt[0] + totalTokens[1] += tt[1] except Exception as e: traceback.print_exc() return [data, totalTokens, e] + finally: + save_progress_yaml(data, filename) return [data, totalTokens, None] @@ -356,16 +375,17 @@ def parseCommonEvents(data, filename): with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: pbar.desc = filename - with ThreadPoolExecutor(max_workers=THREADS) as executor: - futures = [executor.submit(searchCodes, page, pbar, [], filename) for page in data if page is not None] - for future in as_completed(futures): + for page in data: + if page is not None: try: - totalTokensFuture = future.result() - totalTokens[0] += totalTokensFuture[0] - totalTokens[1] += totalTokensFuture[1] + tt = searchCodes(page, pbar, [], filename) + totalTokens[0] += tt[0] + totalTokens[1] += tt[1] except Exception as e: traceback.print_exc() return [data, totalTokens, e] + finally: + save_progress_yaml(data, filename) return [data, totalTokens, None] @@ -384,16 +404,17 @@ def parseTroops(data, filename): pbar.desc = filename for troop in data: if troop is not None: - with ThreadPoolExecutor(max_workers=THREADS) as executor: - futures = [executor.submit(searchCodes, page, pbar, [], filename) for page in troop["pages"] if page is not None] - for future in as_completed(futures): + for page in troop["pages"]: + if page is not None: try: - totalTokensFuture = future.result() - totalTokens[0] += totalTokensFuture[0] - totalTokens[1] += totalTokensFuture[1] + tt = searchCodes(page, pbar, [], filename) + totalTokens[0] += tt[0] + totalTokens[1] += tt[1] except Exception as e: traceback.print_exc() return [data, totalTokens, e] + finally: + save_progress_yaml(data, filename) return [data, totalTokens, None] @@ -411,6 +432,8 @@ def parseNames(data, filename, context): except Exception as e: traceback.print_exc() return [data, totalTokens, e] + finally: + save_progress_yaml(data, filename) return [data, totalTokens, None] @@ -430,6 +453,8 @@ def parseSS(data, filename): except Exception as e: traceback.print_exc() return [data, totalTokens, e] + finally: + save_progress_yaml(data, filename) return [data, totalTokens, None] @@ -456,6 +481,8 @@ def parseSystem(data, filename): except Exception as e: traceback.print_exc() return [data, totalTokens, e] + finally: + save_progress_yaml(data, filename) return [data, totalTokens, None] @@ -470,16 +497,17 @@ def parseScenario(data, filename): with tqdm(bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE) as pbar: pbar.desc = filename - with ThreadPoolExecutor(max_workers=THREADS) as executor: - futures = [executor.submit(searchCodes, page[1], pbar, [], filename) for page in data.items() if page[1] is not None] - for future in as_completed(futures): + for page in data.items(): + if page[1] is not None: try: - totalTokensFuture = future.result() - totalTokens[0] += totalTokensFuture[0] - totalTokens[1] += totalTokensFuture[1] + tt = searchCodes(page[1], pbar, [], filename) + totalTokens[0] += tt[0] + totalTokens[1] += tt[1] except Exception as e: traceback.print_exc() return [data, totalTokens, e] + finally: + save_progress_yaml(data, filename) return [data, totalTokens, None] diff --git a/modules/rpgmakerplugin.py b/modules/rpgmakerplugin.py index 5844f7b..cf49ebf 100644 --- a/modules/rpgmakerplugin.py +++ b/modules/rpgmakerplugin.py @@ -14,6 +14,7 @@ from dotenv import load_dotenv from retry import retry from tqdm import tqdm from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost +import tempfile # Open AI load_dotenv() @@ -176,6 +177,27 @@ def parsePlugin(readFile, filename): return [data, totalTokens, None] +def save_progress_lines(lines, filename, encoding="utf_8"): + """Atomically save current line-based translation progress.""" + try: + if ESTIMATE: + return + os.makedirs("translated", exist_ok=True) + tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") + try: + with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: + tmp_file.writelines(lines) + os.replace(tmp_path, os.path.join("translated", filename)) + finally: + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + except Exception: + traceback.print_exc() + + def translatePlugin(data, pbar, filename, translatedList): if len(translatedList) > 0: questList = translatedList[0] @@ -259,6 +281,7 @@ def translatePlugin(data, pbar, filename, translatedList): with open("translations.txt", "a+", encoding="utf-8") as tlFile: tlFile.write(f"{originalString} ({translatedText})\n") data[i] = data[i].replace(originalString, translatedText) + save_progress_lines(data, filename) # Quest Name regex = r'[\\]+"QuestName[\\]+":[\\]+"(.*?)[\\]+"' @@ -295,6 +318,7 @@ def translatePlugin(data, pbar, filename, translatedList): # Set Data data[i] = data[i].replace(originalString, translatedText) + save_progress_lines(data, filename) # Quest Client regex = r'QuestClientName[\\]+":[\\]+"(.*?)[\\]+"' @@ -328,6 +352,7 @@ def translatePlugin(data, pbar, filename, translatedList): # Set Data data[i] = data[i].replace(originalString, translatedText) + save_progress_lines(data, filename) # Quest Location regex = r'QuestLocation[\\]+":[\\]+"(.*?)[\\]+"' @@ -361,6 +386,7 @@ def translatePlugin(data, pbar, filename, translatedList): # Set Data data[i] = data[i].replace(originalString, translatedText) + save_progress_lines(data, filename) # Quest Target regex = r'PlaceInformation[\\]+":[\\]+"(.*?)[\\]+"' @@ -394,6 +420,7 @@ def translatePlugin(data, pbar, filename, translatedList): # Set Data data[i] = data[i].replace(originalString, translatedText) + save_progress_lines(data, filename) # Quest Summary regex = r'[\\]+"QuestContent[\\]+":[\\]+"[\\]+"(.*?)[\\]+"[\\]+"' @@ -436,6 +463,7 @@ def translatePlugin(data, pbar, filename, translatedList): # Set Data data[i] = data[i].replace(originalString, translatedText) + save_progress_lines(data, filename) # Quest Goal 1 regex = r'ObjectiveContent[\\]+":[\\]+"[\\]+"(.*?)[\\]+"' @@ -476,6 +504,7 @@ def translatePlugin(data, pbar, filename, translatedList): # Set Data data[i] = data[i].replace(originalString, translatedText) + save_progress_lines(data, filename) # Next Line i += 1 diff --git a/modules/text.py b/modules/text.py index 8dc19b9..40eb24a 100644 --- a/modules/text.py +++ b/modules/text.py @@ -14,6 +14,7 @@ from dotenv import load_dotenv from retry import retry from tqdm import tqdm from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost +import tempfile # Open AI load_dotenv() @@ -161,7 +162,7 @@ def parseText(readFile, filename): PBAR = pbar try: - result = translateTxt(data, []) + result = translateTxt(data, filename, []) totalTokens[0] += result[0] totalTokens[1] += result[1] except Exception as e: @@ -170,7 +171,28 @@ def parseText(readFile, filename): return [data, totalTokens, None] -def translateTxt(data, translatedList): +def save_progress_lines(lines, filename, encoding="utf-8"): + """Atomically save current line-based translation progress.""" + try: + if ESTIMATE: + return + os.makedirs("translated", exist_ok=True) + tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") + try: + with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: + tmp_file.writelines(lines) + os.replace(tmp_path, os.path.join("translated", filename)) + finally: + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + except Exception: + traceback.print_exc() + + +def translateTxt(data, filename, translatedList): if translatedList: stringList = translatedList[0] choiceList = translatedList[1] @@ -224,6 +246,7 @@ def translateTxt(data, translatedList): # Set Data data[i] = f"{translatedText}\n" + save_progress_lines(data, filename) i += 1 else: @@ -263,7 +286,7 @@ def translateTxt(data, translatedList): MISMATCH.append(FILENAME) # Set Strings - translateTxt(data, [stringListTL, choiceListTL]) + translateTxt(data, filename, [stringListTL, choiceListTL]) return tokens # Save some money and enter the character before translation diff --git a/modules/tyrano.py b/modules/tyrano.py index babc6f1..2573946 100644 --- a/modules/tyrano.py +++ b/modules/tyrano.py @@ -14,6 +14,7 @@ from dotenv import load_dotenv from retry import retry from tqdm import tqdm from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost +import tempfile # Open AI load_dotenv() @@ -161,7 +162,7 @@ def parseTyrano(readFile, filename): PBAR = pbar try: - result = translateTyrano(data, []) + result = translateTyrano(data, filename, []) totalTokens[0] += result[0] totalTokens[1] += result[1] except Exception as e: @@ -170,7 +171,28 @@ def parseTyrano(readFile, filename): return [data, totalTokens, None] -def translateTyrano(data, translatedList): +def save_progress_lines(lines, filename, encoding="utf-8"): + """Atomically save current line-based translation progress.""" + try: + if ESTIMATE: + return + os.makedirs("translated", exist_ok=True) + tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") + try: + with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: + tmp_file.writelines(lines) + os.replace(tmp_path, os.path.join("translated", filename)) + finally: + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + except Exception: + traceback.print_exc() + + +def translateTyrano(data, filename, translatedList): if translatedList: stringList = translatedList[0] choiceList = translatedList[1] @@ -291,6 +313,8 @@ def translateTyrano(data, translatedList): # Set Data data[i] = data[i].replace(originalString, translatedText) + # Save progress after each line change + save_progress_lines(data, filename) # Choices match = re.search(choiceRegex, data[i]) @@ -311,6 +335,7 @@ def translateTyrano(data, translatedList): # Set data[i] = data[i].replace(match.group(1), translatedText) + save_progress_lines(data, filename) i += 1 else: @@ -350,7 +375,7 @@ def translateTyrano(data, translatedList): MISMATCH.append(FILENAME) # Set Strings - translateTyrano(data, [stringListTL, choiceListTL]) + translateTyrano(data, filename, [stringListTL, choiceListTL]) return tokens # Save some money and enter the character before translation diff --git a/modules/unity.py b/modules/unity.py index c9b7b3b..90f2346 100644 --- a/modules/unity.py +++ b/modules/unity.py @@ -14,6 +14,7 @@ from dotenv import load_dotenv from retry import retry from tqdm import tqdm from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost, getPricingConfig, calculateCost +import tempfile # Open AI load_dotenv() @@ -184,6 +185,27 @@ def parseUnity(readFile, filename): return [data, totalTokens, None] +def save_progress_lines(lines, filename, encoding="utf-8"): + """Atomically save current line-based translation progress.""" + try: + if ESTIMATE: + return + os.makedirs("translated", exist_ok=True) + tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") + try: + with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: + tmp_file.writelines(lines) + os.replace(tmp_path, os.path.join("translated", filename)) + finally: + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + except Exception: + traceback.print_exc() + + def translateUnity(data, pbar, filename, translatedList): stringList = [] tokens = [0, 0] @@ -237,6 +259,7 @@ def translateUnity(data, pbar, filename, translatedList): # Set Data data[i] = f"{leftString}={translatedText}\n" + save_progress_lines(data, filename) i += 1 # Nothing relevant. Skip Line. diff --git a/modules/wolf.py b/modules/wolf.py index 746008d..4fcab32 100644 --- a/modules/wolf.py +++ b/modules/wolf.py @@ -8,7 +8,7 @@ import time import traceback import tiktoken import openai -from concurrent.futures import ThreadPoolExecutor, as_completed +# Removed concurrent.futures usage for simplicity; running synchronously from pathlib import Path from colorama import Fore from dotenv import load_dotenv @@ -190,6 +190,21 @@ def getResultString(translatedData, translationTime, filename): return filename + ": " + totalTokenstring + timeString + Fore.RED + " \u2717 " + errorString + Fore.RESET +def save_progress_json(data, filename): + """Atomically write current JSON data to translated/filename; skip in estimate mode.""" + try: + if ESTIMATE: + return + os.makedirs("translated", exist_ok=True) + tmp_path = os.path.join("translated", f"{filename}.tmp") + final_path = os.path.join("translated", filename) + with open(tmp_path, "w", encoding="utf-8", newline="\n") as outFile: + json.dump(data, outFile, ensure_ascii=False, indent=4) + os.replace(tmp_path, final_path) + except Exception: + traceback.print_exc() + + def parseOther(data, filename): totalTokens = [0, 0] totalLines = 0 @@ -206,6 +221,8 @@ def parseOther(data, filename): totalTokens[1] += translationData[1] except Exception as e: return [data, totalTokens, e] + finally: + save_progress_json(data, filename) return [data, totalTokens, None] @@ -225,6 +242,8 @@ def parseDB(data, filename): totalTokens[1] += translationData[1] except Exception as e: return [data, totalTokens, e] + finally: + save_progress_json(data, filename) return [data, totalTokens, None] @@ -240,21 +259,22 @@ def parseMap(data, filename): for page in event["pages"]: totalLines += len(page["list"]) - # Thread for each page in file + # Process pages synchronously and persist after each with tqdm(bar_format=BAR_FORMAT, position=POSITION, total=totalLines, leave=LEAVE) as pbar: pbar.desc = filename pbar.total = totalLines - with ThreadPoolExecutor(max_workers=THREADS) as executor: - for event in events: - if event is not None: - futures = [executor.submit(searchCodes, page["list"], pbar, None, filename) for page in event["pages"] if page is not None] - for future in as_completed(futures): + for event in events: + if event is not None: + for page in event["pages"]: + if page is not None: try: - totalTokensFuture = future.result() - totalTokens[0] += totalTokensFuture[0] - totalTokens[1] += totalTokensFuture[1] + tt = searchCodes(page["list"], pbar, None, filename) + totalTokens[0] += tt[0] + totalTokens[1] += tt[1] except Exception as e: return [data, totalTokens, e] + finally: + save_progress_json(data, filename) return [data, totalTokens, None] diff --git a/modules/wolf2.py b/modules/wolf2.py index ed59ee1..960d174 100644 --- a/modules/wolf2.py +++ b/modules/wolf2.py @@ -14,6 +14,7 @@ from dotenv import load_dotenv from retry import retry from tqdm import tqdm from util.translation import TranslationConfig, translateAI as sharedtranslateAI, getPricingConfig, calculateCost +import tempfile # Open AI load_dotenv() @@ -175,6 +176,27 @@ def parseWOLF(readFile, filename): return [data, totalTokens, None] +def save_progress_lines(lines, filename, encoding="shift_jis"): + """Atomically save current line-based translation progress.""" + try: + if ESTIMATE: + return + os.makedirs("translated", exist_ok=True) + tmp_fd, tmp_path = tempfile.mkstemp(prefix=f"{filename}.", suffix=".tmp", dir="translated") + try: + with os.fdopen(tmp_fd, "w", encoding=encoding, newline="\n", errors="ignore") as tmp_file: + tmp_file.writelines(lines) + os.replace(tmp_path, os.path.join("translated", filename)) + finally: + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + except Exception: + traceback.print_exc() + + def translateWOLF(data, translatedList, pbar, filename): stringList = [] currentGroup = [] @@ -193,6 +215,7 @@ def translateWOLF(data, translatedList, pbar, filename): tokens[0] += response[1][0] tokens[1] += response[1][1] data[i] = data[i].replace(matchList[0], f"{speaker}") + save_progress_lines(data, filename) i += 1 else: speaker = "" @@ -221,6 +244,7 @@ def translateWOLF(data, translatedList, pbar, filename): data[i] = f"//{choiceListTL[0]}\n" choiceListTL.pop(0) i += 1 + save_progress_lines(data, filename) # Mismatch else: @@ -278,6 +302,7 @@ def translateWOLF(data, translatedList, pbar, filename): # Set Data data.insert(i, f"{translatedText}\n") + save_progress_lines(data, filename) i += 1 # Nothing relevant. Skip Line.