Speed up save functionality
This commit is contained in:
parent
b9e10047a5
commit
9fa3c32b7a
8 changed files with 153 additions and 44 deletions
|
|
@ -260,6 +260,17 @@ def save_progress_yaml(data, filename):
|
|||
traceback.print_exc()
|
||||
|
||||
|
||||
def maybe_save_progress_yaml(data, filename, tokens):
|
||||
"""Save YAML progress only when tokens indicate actual translation work."""
|
||||
try:
|
||||
if not tokens:
|
||||
return
|
||||
if isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1]):
|
||||
save_progress_yaml(data, filename)
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
def parseMap(data, filename):
|
||||
totalTokens = [0, 0]
|
||||
totalLines = 0
|
||||
|
|
@ -297,7 +308,7 @@ def parseMap(data, filename):
|
|||
traceback.print_exc()
|
||||
return [data, totalTokens, e]
|
||||
finally:
|
||||
save_progress_yaml(data, filename)
|
||||
maybe_save_progress_yaml(data, filename, tt)
|
||||
return [data, totalTokens, None]
|
||||
|
||||
|
||||
|
|
@ -385,7 +396,7 @@ def parseCommonEvents(data, filename):
|
|||
traceback.print_exc()
|
||||
return [data, totalTokens, e]
|
||||
finally:
|
||||
save_progress_yaml(data, filename)
|
||||
maybe_save_progress_yaml(data, filename, tt)
|
||||
return [data, totalTokens, None]
|
||||
|
||||
|
||||
|
|
@ -467,7 +478,7 @@ def parseNames(data, filename, context):
|
|||
traceback.print_exc()
|
||||
return [data, totalTokens, e]
|
||||
finally:
|
||||
save_progress_yaml(data, filename)
|
||||
maybe_save_progress_yaml(data, filename, result)
|
||||
return [data, totalTokens, None]
|
||||
|
||||
|
||||
|
|
@ -503,7 +514,7 @@ def parseSS(data, filename):
|
|||
traceback.print_exc()
|
||||
return [data, totalTokens, e]
|
||||
finally:
|
||||
save_progress_yaml(data, filename)
|
||||
maybe_save_progress_yaml(data, filename, result)
|
||||
return [data, totalTokens, None]
|
||||
|
||||
|
||||
|
|
@ -541,7 +552,7 @@ def parseSystem(data, filename):
|
|||
traceback.print_exc()
|
||||
return [data, totalTokens, e]
|
||||
finally:
|
||||
save_progress_yaml(data, filename)
|
||||
maybe_save_progress_yaml(data, filename, result)
|
||||
return [data, totalTokens, None]
|
||||
|
||||
|
||||
|
|
@ -566,7 +577,7 @@ def parseScenario(data, filename):
|
|||
traceback.print_exc()
|
||||
return [data, totalTokens, e]
|
||||
finally:
|
||||
save_progress_yaml(data, filename)
|
||||
maybe_save_progress_yaml(data, filename, tt)
|
||||
return [data, totalTokens, None]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -308,6 +308,20 @@ def saveProgress(data, filename):
|
|||
traceback.print_exc()
|
||||
|
||||
|
||||
def checkSave(data, filename, tokens):
|
||||
"""Save progress only if the given tokens reflect an actual translation.
|
||||
tokens should be a [input_tokens, output_tokens] pair returned by a search/translate call.
|
||||
"""
|
||||
try:
|
||||
if not tokens:
|
||||
return
|
||||
if (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])):
|
||||
saveProgress(data, filename)
|
||||
except Exception:
|
||||
# Don't let saving issues affect the translation flow
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
def update_vocab_section(category: str, pairs: list[tuple[str, str]]):
|
||||
"""Update or insert a section in vocab.txt for the given category with provided pairs.
|
||||
Only writes when there's an actual translation (dst is non-empty and differs from src after normalization).
|
||||
|
|
@ -493,8 +507,8 @@ def parseMap(data, filename):
|
|||
return [data, totalTokens, e]
|
||||
finally:
|
||||
pbar.update(len(page.get("list", [])))
|
||||
# Persist progress after each page
|
||||
saveProgress(data, filename)
|
||||
# Persist progress only if this page produced tokens
|
||||
checkSave(data, filename, totalTokensPage)
|
||||
return [data, totalTokens, None]
|
||||
|
||||
|
||||
|
|
@ -583,8 +597,8 @@ def parseCommonEvents(data, filename):
|
|||
return [data, totalTokens, e]
|
||||
finally:
|
||||
pbar.update(len(page.get("list", [])))
|
||||
# Persist progress after each page
|
||||
saveProgress(data, filename)
|
||||
# Persist progress only if this page produced tokens
|
||||
checkSave(data, filename, totalTokensPage)
|
||||
return [data, totalTokens, None]
|
||||
|
||||
|
||||
|
|
@ -615,8 +629,8 @@ def parseTroops(data, filename):
|
|||
return [data, totalTokens, e]
|
||||
finally:
|
||||
pbar.update(len(page.get("list", [])))
|
||||
# Persist progress after each page
|
||||
saveProgress(data, filename)
|
||||
# Persist progress only if this page produced tokens
|
||||
checkSave(data, filename, totalTokensPage)
|
||||
return [data, totalTokens, None]
|
||||
|
||||
|
||||
|
|
@ -678,8 +692,8 @@ def parseNames(data, filename, context):
|
|||
traceback.print_exc()
|
||||
return [data, totalTokens, e]
|
||||
finally:
|
||||
# Persist progress after completing names pass/batches
|
||||
saveProgress(data, filename)
|
||||
# Persist progress only if this names pass produced tokens
|
||||
checkSave(data, filename, totalTokens)
|
||||
return [data, totalTokens, None]
|
||||
|
||||
|
||||
|
|
@ -715,8 +729,8 @@ def parseSS(data, filename):
|
|||
traceback.print_exc()
|
||||
return [data, totalTokens, e]
|
||||
finally:
|
||||
# Persist progress after each state
|
||||
saveProgress(data, filename)
|
||||
# Persist progress only if this state produced tokens
|
||||
checkSave(data, filename, result)
|
||||
return [data, totalTokens, None]
|
||||
|
||||
|
||||
|
|
@ -757,8 +771,8 @@ def parseSystem(data, filename):
|
|||
traceback.print_exc()
|
||||
return [data, totalTokens, e]
|
||||
finally:
|
||||
# Persist after system sections processed
|
||||
saveProgress(data, filename)
|
||||
# Persist only if system sections produced tokens
|
||||
checkSave(data, filename, result)
|
||||
return [data, totalTokens, None]
|
||||
|
||||
|
||||
|
|
@ -784,8 +798,8 @@ def parseScenario(data, filename):
|
|||
return [data, totalTokens, e]
|
||||
finally:
|
||||
pbar.update(len(page[1]))
|
||||
# Persist progress after each page
|
||||
saveProgress(data, filename)
|
||||
# Persist progress only if this page produced tokens
|
||||
checkSave(data, filename, totalTokensPage)
|
||||
return [data, totalTokens, None]
|
||||
|
||||
|
||||
|
|
@ -987,11 +1001,15 @@ def searchNames(data, pbar, context, filename):
|
|||
if batchFull == True or i >= len(data):
|
||||
k = j # Original Index
|
||||
if context in "Actors":
|
||||
# Track tokens for this batch
|
||||
batchTokens = [0, 0]
|
||||
# Name
|
||||
response = translateAI(nameList, newContext, True)
|
||||
translatedNameBatch = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
batchTokens[0] += response[1][0]
|
||||
batchTokens[1] += response[1][1]
|
||||
if pbar is not None and nameList:
|
||||
pbar.update(len(nameList))
|
||||
pbar.refresh()
|
||||
|
|
@ -1002,6 +1020,8 @@ def searchNames(data, pbar, context, filename):
|
|||
translatedNicknameBatch = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
batchTokens[0] += response[1][0]
|
||||
batchTokens[1] += response[1][1]
|
||||
if pbar is not None:
|
||||
pbar.update(len(nicknameList))
|
||||
pbar.refresh()
|
||||
|
|
@ -1012,6 +1032,8 @@ def searchNames(data, pbar, context, filename):
|
|||
translatedProfileBatch = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
batchTokens[0] += response[1][0]
|
||||
batchTokens[1] += response[1][1]
|
||||
if pbar is not None:
|
||||
pbar.update(len(profileList))
|
||||
pbar.refresh()
|
||||
|
|
@ -1047,17 +1069,21 @@ def searchNames(data, pbar, context, filename):
|
|||
batchFull = False
|
||||
filling = False
|
||||
j += 1
|
||||
# Persist after applying this batch
|
||||
saveProgress(data, filename)
|
||||
# Persist after applying this batch only if we actually translated something in this batch
|
||||
checkSave(data, filename, batchTokens)
|
||||
else:
|
||||
mismatch = True
|
||||
|
||||
if context in ["Armors", "Weapons", "Items", "Skills"]:
|
||||
# Track tokens for this batch
|
||||
batchTokens = [0, 0]
|
||||
# Name
|
||||
response = translateAI(nameList, newContext, True)
|
||||
translatedNameBatch = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
batchTokens[0] += response[1][0]
|
||||
batchTokens[1] += response[1][1]
|
||||
if pbar is not None and nameList:
|
||||
pbar.update(len(nameList))
|
||||
pbar.refresh()
|
||||
|
|
@ -1072,6 +1098,8 @@ def searchNames(data, pbar, context, filename):
|
|||
translatedDescriptionBatch = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
batchTokens[0] += response[1][0]
|
||||
batchTokens[1] += response[1][1]
|
||||
if pbar is not None:
|
||||
pbar.update(len(descriptionList))
|
||||
pbar.refresh()
|
||||
|
|
@ -1107,15 +1135,19 @@ def searchNames(data, pbar, context, filename):
|
|||
batchFull = False
|
||||
filling = False
|
||||
j += 1
|
||||
# Persist after applying this batch
|
||||
saveProgress(data, filename)
|
||||
# Persist after applying this batch only if we actually translated something in this batch
|
||||
checkSave(data, filename, batchTokens)
|
||||
else:
|
||||
mismatch = True
|
||||
if context in ["Enemies", "Classes", "MapInfos"]:
|
||||
# Track tokens for this batch
|
||||
batchTokens = [0, 0]
|
||||
response = translateAI(nameList, newContext, True)
|
||||
translatedNameBatch = response[0]
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
batchTokens[0] += response[1][0]
|
||||
batchTokens[1] += response[1][1]
|
||||
if pbar is not None and nameList:
|
||||
pbar.update(len(nameList))
|
||||
pbar.refresh()
|
||||
|
|
@ -1146,8 +1178,8 @@ def searchNames(data, pbar, context, filename):
|
|||
batchFull = False
|
||||
filling = False
|
||||
j += 1
|
||||
# Persist after applying this batch
|
||||
saveProgress(data, filename)
|
||||
# Persist after applying this batch only if we actually translated something in this batch
|
||||
checkSave(data, filename, batchTokens)
|
||||
else:
|
||||
mismatch = True
|
||||
|
||||
|
|
|
|||
|
|
@ -198,6 +198,17 @@ def save_progress_lines(lines, filename, encoding="utf_8"):
|
|||
traceback.print_exc()
|
||||
|
||||
|
||||
def saveCheckLines(lines, filename, tokens=None, encoding="utf_8"):
|
||||
"""Save progress only when tokens indicate work or when explicitly called after a mutation."""
|
||||
try:
|
||||
if tokens is not None:
|
||||
if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])):
|
||||
return
|
||||
save_progress_lines(lines, filename, encoding=encoding)
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
def translatePlugin(data, pbar, filename, translatedList):
|
||||
if len(translatedList) > 0:
|
||||
questList = translatedList[0]
|
||||
|
|
@ -281,7 +292,7 @@ def translatePlugin(data, pbar, filename, translatedList):
|
|||
with open("translations.txt", "a+", encoding="utf-8") as tlFile:
|
||||
tlFile.write(f"{originalString} ({translatedText})\n")
|
||||
data[i] = data[i].replace(originalString, translatedText)
|
||||
save_progress_lines(data, filename)
|
||||
saveCheckLines(data, filename)
|
||||
|
||||
# Quest Name
|
||||
regex = r'[\\]+"QuestName[\\]+":[\\]+"(.*?)[\\]+"'
|
||||
|
|
@ -318,7 +329,7 @@ def translatePlugin(data, pbar, filename, translatedList):
|
|||
|
||||
# Set Data
|
||||
data[i] = data[i].replace(originalString, translatedText)
|
||||
save_progress_lines(data, filename)
|
||||
saveCheckLines(data, filename)
|
||||
|
||||
# Quest Client
|
||||
regex = r'QuestClientName[\\]+":[\\]+"(.*?)[\\]+"'
|
||||
|
|
@ -352,7 +363,7 @@ def translatePlugin(data, pbar, filename, translatedList):
|
|||
|
||||
# Set Data
|
||||
data[i] = data[i].replace(originalString, translatedText)
|
||||
save_progress_lines(data, filename)
|
||||
saveCheckLines(data, filename)
|
||||
|
||||
# Quest Location
|
||||
regex = r'QuestLocation[\\]+":[\\]+"(.*?)[\\]+"'
|
||||
|
|
@ -386,7 +397,7 @@ def translatePlugin(data, pbar, filename, translatedList):
|
|||
|
||||
# Set Data
|
||||
data[i] = data[i].replace(originalString, translatedText)
|
||||
save_progress_lines(data, filename)
|
||||
saveCheckLines(data, filename)
|
||||
|
||||
# Quest Target
|
||||
regex = r'PlaceInformation[\\]+":[\\]+"(.*?)[\\]+"'
|
||||
|
|
@ -420,7 +431,7 @@ def translatePlugin(data, pbar, filename, translatedList):
|
|||
|
||||
# Set Data
|
||||
data[i] = data[i].replace(originalString, translatedText)
|
||||
save_progress_lines(data, filename)
|
||||
saveCheckLines(data, filename)
|
||||
|
||||
# Quest Summary
|
||||
regex = r'[\\]+"QuestContent[\\]+":[\\]+"[\\]+"(.*?)[\\]+"[\\]+"'
|
||||
|
|
@ -463,7 +474,7 @@ def translatePlugin(data, pbar, filename, translatedList):
|
|||
|
||||
# Set Data
|
||||
data[i] = data[i].replace(originalString, translatedText)
|
||||
save_progress_lines(data, filename)
|
||||
saveCheckLines(data, filename)
|
||||
|
||||
# Quest Goal 1
|
||||
regex = r'ObjectiveContent[\\]+":[\\]+"[\\]+"(.*?)[\\]+"'
|
||||
|
|
@ -504,7 +515,7 @@ def translatePlugin(data, pbar, filename, translatedList):
|
|||
|
||||
# Set Data
|
||||
data[i] = data[i].replace(originalString, translatedText)
|
||||
save_progress_lines(data, filename)
|
||||
saveCheckLines(data, filename)
|
||||
|
||||
# Next Line
|
||||
i += 1
|
||||
|
|
|
|||
|
|
@ -192,6 +192,17 @@ def save_progress_lines(lines, filename, encoding="utf-8"):
|
|||
traceback.print_exc()
|
||||
|
||||
|
||||
def saveCheckLines(lines, filename, tokens=None, encoding="utf-8"):
|
||||
"""Save progress only when tokens indicate work or when explicitly called after a mutation."""
|
||||
try:
|
||||
if tokens is not None:
|
||||
if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])):
|
||||
return
|
||||
save_progress_lines(lines, filename, encoding=encoding)
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
def translateTxt(data, filename, translatedList):
|
||||
if translatedList:
|
||||
stringList = translatedList[0]
|
||||
|
|
@ -246,7 +257,7 @@ def translateTxt(data, filename, translatedList):
|
|||
|
||||
# Set Data
|
||||
data[i] = f"{translatedText}\n"
|
||||
save_progress_lines(data, filename)
|
||||
saveCheckLines(data, filename)
|
||||
|
||||
i += 1
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -192,6 +192,18 @@ def save_progress_lines(lines, filename, encoding="utf-8"):
|
|||
traceback.print_exc()
|
||||
|
||||
|
||||
def saveCheckLines(lines, filename, tokens=None, encoding="utf-8"):
|
||||
"""Save progress only when tokens indicate work or when tokens is None but we know a mutation occurred."""
|
||||
try:
|
||||
# If explicit tokens provided, gate on tokens; otherwise assume we were called on actual mutation
|
||||
if tokens is not None:
|
||||
if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])):
|
||||
return
|
||||
save_progress_lines(lines, filename, encoding=encoding)
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
def translateTyrano(data, filename, translatedList):
|
||||
if translatedList:
|
||||
stringList = translatedList[0]
|
||||
|
|
@ -313,8 +325,8 @@ def translateTyrano(data, filename, translatedList):
|
|||
|
||||
# Set Data
|
||||
data[i] = data[i].replace(originalString, translatedText)
|
||||
# Save progress after each line change
|
||||
save_progress_lines(data, filename)
|
||||
# Save progress after each line change (we mutated, so tokens gate optional)
|
||||
saveCheckLines(data, filename)
|
||||
|
||||
# Choices
|
||||
match = re.search(choiceRegex, data[i])
|
||||
|
|
@ -335,7 +347,7 @@ def translateTyrano(data, filename, translatedList):
|
|||
|
||||
# Set
|
||||
data[i] = data[i].replace(match.group(1), translatedText)
|
||||
save_progress_lines(data, filename)
|
||||
saveCheckLines(data, filename)
|
||||
|
||||
i += 1
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -206,6 +206,17 @@ def save_progress_lines(lines, filename, encoding="utf-8"):
|
|||
traceback.print_exc()
|
||||
|
||||
|
||||
def saveCheckLines(lines, filename, tokens=None, encoding="utf-8"):
|
||||
"""Save progress only when tokens indicate work or when explicitly called after a mutation."""
|
||||
try:
|
||||
if tokens is not None:
|
||||
if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])):
|
||||
return
|
||||
save_progress_lines(lines, filename, encoding=encoding)
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
def translateUnity(data, pbar, filename, translatedList):
|
||||
stringList = []
|
||||
tokens = [0, 0]
|
||||
|
|
@ -259,7 +270,7 @@ def translateUnity(data, pbar, filename, translatedList):
|
|||
|
||||
# Set Data
|
||||
data[i] = f"{leftString}={translatedText}\n"
|
||||
save_progress_lines(data, filename)
|
||||
saveCheckLines(data, filename)
|
||||
i += 1
|
||||
|
||||
# Nothing relevant. Skip Line.
|
||||
|
|
|
|||
|
|
@ -205,6 +205,17 @@ def save_progress_json(data, filename):
|
|||
traceback.print_exc()
|
||||
|
||||
|
||||
def maybe_save_progress_json(data, filename, tokens):
|
||||
"""Save JSON progress only when tokens indicate actual translation work."""
|
||||
try:
|
||||
if not tokens:
|
||||
return
|
||||
if isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1]):
|
||||
save_progress_json(data, filename)
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
def parseOther(data, filename):
|
||||
totalTokens = [0, 0]
|
||||
totalLines = 0
|
||||
|
|
@ -222,7 +233,7 @@ def parseOther(data, filename):
|
|||
except Exception as e:
|
||||
return [data, totalTokens, e]
|
||||
finally:
|
||||
save_progress_json(data, filename)
|
||||
maybe_save_progress_json(data, filename, translationData)
|
||||
return [data, totalTokens, None]
|
||||
|
||||
|
||||
|
|
@ -243,7 +254,7 @@ def parseDB(data, filename):
|
|||
except Exception as e:
|
||||
return [data, totalTokens, e]
|
||||
finally:
|
||||
save_progress_json(data, filename)
|
||||
maybe_save_progress_json(data, filename, translationData)
|
||||
return [data, totalTokens, None]
|
||||
|
||||
|
||||
|
|
@ -274,7 +285,7 @@ def parseMap(data, filename):
|
|||
except Exception as e:
|
||||
return [data, totalTokens, e]
|
||||
finally:
|
||||
save_progress_json(data, filename)
|
||||
maybe_save_progress_json(data, filename, tt)
|
||||
return [data, totalTokens, None]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -222,6 +222,16 @@ def save_progress_lines(lines, filename, encoding="shift_jis"):
|
|||
traceback.print_exc()
|
||||
|
||||
|
||||
def saveCheckLines(lines, filename, tokens=None, encoding="shift_jis"):
|
||||
try:
|
||||
if tokens is not None:
|
||||
if not (isinstance(tokens, (list, tuple)) and len(tokens) >= 2 and (tokens[0] or tokens[1])):
|
||||
return
|
||||
save_progress_lines(lines, filename, encoding=encoding)
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
def translateWOLF(data, translatedList, pbar, filename):
|
||||
stringList = []
|
||||
currentGroup = []
|
||||
|
|
@ -240,7 +250,7 @@ def translateWOLF(data, translatedList, pbar, filename):
|
|||
tokens[0] += response[1][0]
|
||||
tokens[1] += response[1][1]
|
||||
data[i] = data[i].replace(matchList[0], f"{speaker}")
|
||||
save_progress_lines(data, filename)
|
||||
saveCheckLines(data, filename)
|
||||
i += 1
|
||||
else:
|
||||
speaker = ""
|
||||
|
|
@ -269,7 +279,7 @@ def translateWOLF(data, translatedList, pbar, filename):
|
|||
data[i] = f"//{choiceListTL[0]}\n"
|
||||
choiceListTL.pop(0)
|
||||
i += 1
|
||||
save_progress_lines(data, filename)
|
||||
saveCheckLines(data, filename)
|
||||
|
||||
# Mismatch
|
||||
else:
|
||||
|
|
@ -327,7 +337,7 @@ def translateWOLF(data, translatedList, pbar, filename):
|
|||
|
||||
# Set Data
|
||||
data.insert(i, f"{translatedText}\n")
|
||||
save_progress_lines(data, filename)
|
||||
saveCheckLines(data, filename)
|
||||
i += 1
|
||||
|
||||
# Nothing relevant. Skip Line.
|
||||
|
|
|
|||
Loading…
Reference in a new issue