more accurate progress bar for mvmz + ace
This commit is contained in:
no
parent
384d0930eb
commit
5b6771d88c
2 changed files with 942 additions and 228 deletions
|
|
@ -37,6 +37,7 @@ PBAR = None
|
|||
FILENAME = None
|
||||
TIMETOTAL = 0 # Total Time Taken for all translations
|
||||
VOCAB_LOCK = threading.Lock()
|
||||
PREFLIGHT_COUNT_MODE = False # When True, translateAI wrapper only counts units and never calls API
|
||||
|
||||
# Speakers
|
||||
NAMESLIST = []
|
||||
|
|
@ -447,10 +448,101 @@ def update_vocab_section(category: str, pairs: list[tuple[str, str]]):
|
|||
|
||||
def parseMap(data, filename):
|
||||
totalTokens = [0, 0]
|
||||
totalLines = 0
|
||||
events = data["events"]
|
||||
global LOCK
|
||||
|
||||
# --- Preflight: estimate exact progress total using the same translation batching ---
|
||||
class _CountingBar:
|
||||
def __init__(self):
|
||||
self.n = 0
|
||||
def update(self, n=1):
|
||||
try:
|
||||
self.n += int(n) if n is not None else 1
|
||||
except Exception:
|
||||
self.n += 1
|
||||
def write(self, *args, **kwargs):
|
||||
pass
|
||||
def refresh(self):
|
||||
pass
|
||||
|
||||
def _estimate_map_units(d, fname) -> int:
|
||||
dcopy = copy.deepcopy(d)
|
||||
bar = _CountingBar()
|
||||
global PREFLIGHT_COUNT_MODE
|
||||
saved_preflight = PREFLIGHT_COUNT_MODE
|
||||
global PBAR
|
||||
saved_pbar = PBAR
|
||||
PREFLIGHT_COUNT_MODE = True
|
||||
PBAR = bar
|
||||
try:
|
||||
# Count display name TL (1 unit if present)
|
||||
if "Map" in fname and isinstance(dcopy.get("display_name", None), str):
|
||||
try:
|
||||
translateAI(
|
||||
dcopy["display_name"],
|
||||
"Reply with only the " + LANGUAGE + " translation of the RPG location name",
|
||||
False,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Notes and pages
|
||||
evts = dcopy.get("events", {}) or {}
|
||||
for evt in evts.values():
|
||||
if not evt:
|
||||
continue
|
||||
note_val = evt.get("note") or ""
|
||||
if not isinstance(note_val, str):
|
||||
note_val = str(note_val) if note_val is not None else ""
|
||||
|
||||
# <LB> name translation
|
||||
if "<LB>" in note_val:
|
||||
name_val = evt.get("name") or ""
|
||||
if isinstance(name_val, str) and name_val:
|
||||
try:
|
||||
translateAI(
|
||||
name_val,
|
||||
"Reply with only the " + LANGUAGE + " translation of the RPG location name",
|
||||
False,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# <msgText:"...">
|
||||
if "<msgText:" in note_val:
|
||||
try:
|
||||
translateNote(evt, r"<msgText:\"(.*?)\">", False)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# <namePop:...>, <LB:...>, <dn:...> handled before page processing in real run
|
||||
if "<namePop:" in note_val:
|
||||
try:
|
||||
translateNoteOmitSpace(evt, r"<namePop:\s?([\w一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\uFF61-\uFF9F]+)")
|
||||
except Exception:
|
||||
pass
|
||||
if "<LB:" in note_val:
|
||||
try:
|
||||
translateNoteOmitSpace(evt, r"<LB:(.*?)\s?>.*")
|
||||
except Exception:
|
||||
pass
|
||||
if "<dn:" in note_val:
|
||||
try:
|
||||
translateNoteOmitSpace(evt, r"<dn:\s*(.*)>.*")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for page in (evt.get("pages", []) or []):
|
||||
try:
|
||||
searchCodes(page, bar, [], fname)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return getattr(bar, "n", 0) or 0
|
||||
finally:
|
||||
PREFLIGHT_COUNT_MODE = saved_preflight
|
||||
PBAR = saved_pbar
|
||||
|
||||
# Translate display_name for Map files
|
||||
if "Map" in filename:
|
||||
response = translateAI(
|
||||
|
|
@ -462,30 +554,17 @@ def parseMap(data, filename):
|
|||
totalTokens[1] += response[1][1]
|
||||
data["display_name"] = response[0].replace('"', "")
|
||||
|
||||
# Get total for progress bar (sum of all command list lengths across pages)
|
||||
for event in events.values():
|
||||
if event and isinstance(event, dict):
|
||||
note_val = event.get("note") or ""
|
||||
if not isinstance(note_val, str):
|
||||
note_val = str(note_val) if note_val is not None else ""
|
||||
if "<LB>" in note_val:
|
||||
# Translate event name when flagged with <LB>
|
||||
name_val = event.get("name") or ""
|
||||
if isinstance(name_val, str) and name_val:
|
||||
response = translateAI(
|
||||
name_val,
|
||||
"Reply with only the " + LANGUAGE + " translation of the RPG location name",
|
||||
False,
|
||||
)
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
event["name"] = response[0].replace('"', "")
|
||||
if "<msgText:" in note_val:
|
||||
tokensResponse = translateNote(event, r"<msgText:\"(.*?)\">", False)
|
||||
totalTokens[0] += tokensResponse[0]
|
||||
totalTokens[1] += tokensResponse[1]
|
||||
for page in event["pages"]:
|
||||
totalLines += len(page["list"])
|
||||
totalLines = _estimate_map_units(data, filename)
|
||||
if not isinstance(totalLines, int) or totalLines <= 0:
|
||||
# Fallback to naive count so a bar still renders
|
||||
totalLines = 0
|
||||
for evt in events.values():
|
||||
if evt:
|
||||
for page in (evt.get("pages", []) or []):
|
||||
try:
|
||||
totalLines += len(page.get("list", []))
|
||||
except Exception:
|
||||
pass
|
||||
global PBAR
|
||||
|
||||
# Process each page synchronously with progress updates
|
||||
|
|
@ -609,13 +688,52 @@ def translateNoteOmitSpace(event, regex):
|
|||
|
||||
def parseCommonEvents(data, filename):
|
||||
totalTokens = [0, 0]
|
||||
totalLines = 0
|
||||
global LOCK
|
||||
|
||||
# Get total for progress bar
|
||||
for page in data:
|
||||
if page is not None:
|
||||
totalLines += len(page["list"])
|
||||
# --- Preflight: estimate exact progress total using same batching ---
|
||||
class _CountingBar:
|
||||
def __init__(self):
|
||||
self.n = 0
|
||||
def update(self, n=1):
|
||||
try:
|
||||
self.n += int(n) if n is not None else 1
|
||||
except Exception:
|
||||
self.n += 1
|
||||
def write(self, *args, **kwargs):
|
||||
pass
|
||||
def refresh(self):
|
||||
pass
|
||||
|
||||
def _estimate_units(pages, fname) -> int:
|
||||
dcopy = copy.deepcopy(pages)
|
||||
bar = _CountingBar()
|
||||
global PREFLIGHT_COUNT_MODE, PBAR
|
||||
saved_flag = PREFLIGHT_COUNT_MODE
|
||||
saved_pbar = PBAR
|
||||
PREFLIGHT_COUNT_MODE = True
|
||||
PBAR = bar
|
||||
try:
|
||||
for page in dcopy:
|
||||
if page is not None:
|
||||
try:
|
||||
searchCodes(page, bar, [], fname)
|
||||
except Exception:
|
||||
pass
|
||||
return getattr(bar, "n", 0) or 0
|
||||
finally:
|
||||
PREFLIGHT_COUNT_MODE = saved_flag
|
||||
PBAR = saved_pbar
|
||||
|
||||
totalLines = _estimate_units(data, filename)
|
||||
if not isinstance(totalLines, int) or totalLines <= 0:
|
||||
# Fallback to naive command count
|
||||
totalLines = 0
|
||||
for page in data:
|
||||
if page is not None:
|
||||
try:
|
||||
totalLines += len(page.get("list", []))
|
||||
except Exception:
|
||||
pass
|
||||
global PBAR
|
||||
|
||||
with tqdm(total=totalLines, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar:
|
||||
|
|
@ -637,15 +755,55 @@ def parseCommonEvents(data, filename):
|
|||
|
||||
def parseTroops(data, filename):
|
||||
totalTokens = [0, 0]
|
||||
totalLines = 0
|
||||
global LOCK
|
||||
|
||||
# Get total for progress bar
|
||||
for troop in data:
|
||||
if troop is not None:
|
||||
for page in troop["pages"]:
|
||||
# Progress measured by number of commands in each page's list
|
||||
totalLines += len(page["list"])
|
||||
# --- Preflight total using same code paths ---
|
||||
class _CountingBar:
|
||||
def __init__(self):
|
||||
self.n = 0
|
||||
def update(self, n=1):
|
||||
try:
|
||||
self.n += int(n) if n is not None else 1
|
||||
except Exception:
|
||||
self.n += 1
|
||||
def write(self, *args, **kwargs):
|
||||
pass
|
||||
def refresh(self):
|
||||
pass
|
||||
|
||||
def _estimate_units(troops, fname) -> int:
|
||||
tcopy = copy.deepcopy(troops)
|
||||
bar = _CountingBar()
|
||||
global PREFLIGHT_COUNT_MODE, PBAR
|
||||
saved_flag = PREFLIGHT_COUNT_MODE
|
||||
saved_pbar = PBAR
|
||||
PREFLIGHT_COUNT_MODE = True
|
||||
PBAR = bar
|
||||
try:
|
||||
for troop in tcopy:
|
||||
if troop is None:
|
||||
continue
|
||||
for page in (troop.get("pages", []) or []):
|
||||
if page is not None:
|
||||
try:
|
||||
searchCodes(page, bar, [], fname)
|
||||
except Exception:
|
||||
pass
|
||||
return getattr(bar, "n", 0) or 0
|
||||
finally:
|
||||
PREFLIGHT_COUNT_MODE = saved_flag
|
||||
PBAR = saved_pbar
|
||||
|
||||
totalLines = _estimate_units(data, filename)
|
||||
if not isinstance(totalLines, int) or totalLines <= 0:
|
||||
totalLines = 0
|
||||
for troop in data:
|
||||
if troop is not None:
|
||||
for page in troop.get("pages", []) or []:
|
||||
try:
|
||||
totalLines += len(page.get("list", []))
|
||||
except Exception:
|
||||
pass
|
||||
global PBAR
|
||||
|
||||
with tqdm(total=totalLines, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar:
|
||||
|
|
@ -670,49 +828,138 @@ def parseTroops(data, filename):
|
|||
def parseNames(data, filename, context):
|
||||
totalTokens = [0, 0]
|
||||
|
||||
# Precompute total work units for progress bar
|
||||
def count_work_units(data, context):
|
||||
total = 0
|
||||
# --- Preflight: custom estimator that mirrors searchNames increments (incl. notes/messages) ---
|
||||
class _CountingBar:
|
||||
def __init__(self):
|
||||
self.n = 0
|
||||
def update(self, n=1):
|
||||
try:
|
||||
self.n += int(n) if n is not None else 1
|
||||
except Exception:
|
||||
self.n += 1
|
||||
def write(self, *args, **kwargs):
|
||||
pass
|
||||
def refresh(self):
|
||||
pass
|
||||
|
||||
def _estimate_names_units(entries, ctx, fname) -> int:
|
||||
ecopy = copy.deepcopy(entries)
|
||||
bar = _CountingBar()
|
||||
global PREFLIGHT_COUNT_MODE, PBAR
|
||||
saved_flag = PREFLIGHT_COUNT_MODE
|
||||
saved_pbar = PBAR
|
||||
PREFLIGHT_COUNT_MODE = True
|
||||
PBAR = bar
|
||||
try:
|
||||
# Counts
|
||||
name_cnt = 0
|
||||
desc_cnt = 0
|
||||
profile_cnt = 0
|
||||
nickname_cnt = 0
|
||||
msg_cnt = 0
|
||||
notes_cnt = 0
|
||||
|
||||
note_regexes = [
|
||||
(r"<note:(.*?)>", False),
|
||||
(r"<PE拡張:(.*?)>", False),
|
||||
(r"<hint:(.*?)>", False),
|
||||
(r"<SGDescription:(.*?)>", False),
|
||||
(r"<SG説明:\n?(.*?)>", False),
|
||||
(r"<SG説明2:\n?(.*?)>", False),
|
||||
(r"<SG説明3:\n?(.*?)>", False),
|
||||
(r"<SG説明4:\n?(.*?)>", False),
|
||||
(r"<SG説明:.+?Client\s?:.+?\n\n(.*?)>", True),
|
||||
(r"<SGカテゴリ:(.*?)>", False),
|
||||
(r"<Switch Shop Description>\n(.*)\n", False),
|
||||
(r"<MapText:(.*?)>", False),
|
||||
(r"WATs:(.+?)>", False),
|
||||
(r"ADTs?:(.+?)>", False),
|
||||
(r"<detail:(.*?)>", False),
|
||||
(r"<Name:(.*?)>", False),
|
||||
(r"<sub_1:([^>]+)", True),
|
||||
(r"<sub_2:([^>]+)", True),
|
||||
(r"<sub_3:([^>]+)", True),
|
||||
(r"<infowindow:(.*?)>", True),
|
||||
(r"<ExtendDesc:(.*?)>", True),
|
||||
(r"<desc\d:(.*?)>", False),
|
||||
(r"<拡張説明:(.+?)>", False),
|
||||
(r"<STS DESC>\n(.+?)\n<", False),
|
||||
(r"text:(.+)>", False),
|
||||
]
|
||||
|
||||
for entry in ecopy:
|
||||
if not entry:
|
||||
continue
|
||||
nm = entry.get("name") or ""
|
||||
ds = entry.get("description") or ""
|
||||
nn = entry.get("nickname") or ""
|
||||
pf = entry.get("profile") or ""
|
||||
if ctx == "Actors":
|
||||
if nm: name_cnt += 1
|
||||
if nn: nickname_cnt += 1
|
||||
if pf: profile_cnt += 1
|
||||
elif ctx in ["Armors", "Weapons", "Items"]:
|
||||
if nm: name_cnt += 1
|
||||
if ds: desc_cnt += 1
|
||||
elif ctx == "Skills":
|
||||
if nm: name_cnt += 1
|
||||
if ds: desc_cnt += 1
|
||||
for k in range(1,5):
|
||||
if entry.get(f"message{k}"): msg_cnt += 1
|
||||
elif ctx in ["Enemies", "Classes", "MapInfos"]:
|
||||
if nm: name_cnt += 1
|
||||
|
||||
# Notes counting
|
||||
note = entry.get("note") or ""
|
||||
if isinstance(note, str) and note:
|
||||
for regex, _ww in note_regexes:
|
||||
try:
|
||||
matches = re.findall(regex, note, re.DOTALL)
|
||||
except Exception:
|
||||
matches = []
|
||||
if regex.startswith(r"<SG説明:"):
|
||||
for m in matches:
|
||||
s = m if isinstance(m, str) else (m[0] if m else "")
|
||||
if "Client:" in s or "Client :" in s:
|
||||
continue
|
||||
notes_cnt += 1
|
||||
else:
|
||||
notes_cnt += len(matches)
|
||||
|
||||
# Simulate increments via wrapper (updates bar.n)
|
||||
if name_cnt:
|
||||
translateAI([""] * name_cnt, "", True)
|
||||
if desc_cnt:
|
||||
translateAI([""] * desc_cnt, "", True)
|
||||
if profile_cnt:
|
||||
translateAI([""] * profile_cnt, "", True)
|
||||
if nickname_cnt:
|
||||
translateAI([""] * nickname_cnt, "", True)
|
||||
if msg_cnt:
|
||||
translateAI([""] * msg_cnt, "", True)
|
||||
if notes_cnt:
|
||||
translateAI([""] * notes_cnt, "", True)
|
||||
|
||||
return getattr(bar, "n", 0) or 0
|
||||
finally:
|
||||
PREFLIGHT_COUNT_MODE = saved_flag
|
||||
PBAR = saved_pbar
|
||||
|
||||
total_units = _estimate_names_units(data, context, filename)
|
||||
if not isinstance(total_units, int) or total_units <= 0:
|
||||
# Reasonable fallback: count visible fields/messages (no notes)
|
||||
total_units = 0
|
||||
for entry in data:
|
||||
if not entry:
|
||||
continue
|
||||
|
||||
# Names and associated fields
|
||||
name = entry.get("name", "")
|
||||
desc = entry.get("description", "")
|
||||
nickname = entry.get("nickname", "")
|
||||
profile = entry.get("profile", "")
|
||||
|
||||
if entry.get("name"): total_units += 1
|
||||
if context in ["Armors", "Weapons", "Items", "Skills"] and entry.get("description"): total_units += 1
|
||||
if context == "Actors":
|
||||
if name:
|
||||
total += 1
|
||||
if nickname:
|
||||
total += 1
|
||||
if profile:
|
||||
total += 1
|
||||
elif context in ["Armors", "Weapons", "Items"]:
|
||||
if name:
|
||||
total += 1
|
||||
if desc:
|
||||
total += 1
|
||||
elif context == "Skills":
|
||||
if name:
|
||||
total += 1
|
||||
if desc:
|
||||
total += 1
|
||||
# Messages translated individually in searchNames
|
||||
for n in range(1, 5):
|
||||
msg = entry.get(f"message{n}")
|
||||
if msg:
|
||||
total += 1
|
||||
elif context in ["Enemies", "Classes", "MapInfos"]:
|
||||
if name:
|
||||
total += 1
|
||||
|
||||
return total
|
||||
|
||||
total_units = count_work_units(data, context)
|
||||
if entry.get("nickname"): total_units += 1
|
||||
if entry.get("profile"): total_units += 1
|
||||
if context == "Skills":
|
||||
for k in range(1,5):
|
||||
if entry.get(f"message{k}"): total_units += 1
|
||||
global PBAR
|
||||
|
||||
with tqdm(total=total_units, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar:
|
||||
|
|
@ -734,22 +981,50 @@ def parseNames(data, filename, context):
|
|||
def parseSS(data, filename):
|
||||
totalTokens = [0, 0]
|
||||
|
||||
# Precompute total units (ignore notes): name, description, message1..4 presence
|
||||
def count_work_units(states):
|
||||
total = 0
|
||||
for st in states:
|
||||
# --- Preflight using searchSS over deep copy ---
|
||||
class _CountingBar:
|
||||
def __init__(self):
|
||||
self.n = 0
|
||||
def update(self, n=1):
|
||||
try:
|
||||
self.n += int(n) if n is not None else 1
|
||||
except Exception:
|
||||
self.n += 1
|
||||
def write(self, *args, **kwargs):
|
||||
pass
|
||||
def refresh(self):
|
||||
pass
|
||||
|
||||
def _estimate_units(states, fname) -> int:
|
||||
scopy = copy.deepcopy(states)
|
||||
bar = _CountingBar()
|
||||
global PREFLIGHT_COUNT_MODE, PBAR
|
||||
saved_flag = PREFLIGHT_COUNT_MODE
|
||||
saved_pbar = PBAR
|
||||
PREFLIGHT_COUNT_MODE = True
|
||||
PBAR = bar
|
||||
try:
|
||||
for ss in scopy:
|
||||
if ss is not None:
|
||||
try:
|
||||
searchSS(ss, bar)
|
||||
except Exception:
|
||||
pass
|
||||
return getattr(bar, "n", 0) or 0
|
||||
finally:
|
||||
PREFLIGHT_COUNT_MODE = saved_flag
|
||||
PBAR = saved_pbar
|
||||
|
||||
total_units = _estimate_units(data, filename)
|
||||
if not isinstance(total_units, int) or total_units <= 0:
|
||||
total_units = 0
|
||||
for st in data:
|
||||
if not st:
|
||||
continue
|
||||
if st.get("name"):
|
||||
total += 1
|
||||
if st.get("description"):
|
||||
total += 1
|
||||
for n in range(1, 5):
|
||||
if st.get(f"message{n}"):
|
||||
total += 1
|
||||
return total
|
||||
|
||||
total_units = count_work_units(data)
|
||||
if st.get("name"): total_units += 1
|
||||
if st.get("description"): total_units += 1
|
||||
for n in range(1,5):
|
||||
if st.get(f"message{n}"): total_units += 1
|
||||
global PBAR
|
||||
|
||||
with tqdm(total=total_units, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar:
|
||||
|
|
@ -772,27 +1047,54 @@ def parseSS(data, filename):
|
|||
def parseSystem(data, filename):
|
||||
totalTokens = [0, 0]
|
||||
|
||||
# Precompute total units for Ace schema
|
||||
def count_work_units(sys):
|
||||
total = 0
|
||||
# Terms: sum list lengths of each term list
|
||||
for term in sys.get("terms", {}):
|
||||
termList = sys["terms"][term]
|
||||
if isinstance(termList, list):
|
||||
total += len(termList)
|
||||
# game_title might be a string; count as 1 if non-empty
|
||||
gt = sys.get("game_title")
|
||||
if isinstance(gt, str) and gt:
|
||||
total += 1
|
||||
# variables is a list
|
||||
total += len(sys.get("variables", []) or [])
|
||||
total += len(sys.get("weapon_types", []) or [])
|
||||
total += len(sys.get("armor_types", []) or [])
|
||||
total += len(sys.get("skill_types", []) or [])
|
||||
total += len(sys.get("equip_types", []) or [])
|
||||
return total
|
||||
# --- Preflight: call searchSystem on deep copy to count increments ---
|
||||
class _CountingBar:
|
||||
def __init__(self):
|
||||
self.n = 0
|
||||
def update(self, n=1):
|
||||
try:
|
||||
self.n += int(n) if n is not None else 1
|
||||
except Exception:
|
||||
self.n += 1
|
||||
def write(self, *args, **kwargs):
|
||||
pass
|
||||
def refresh(self):
|
||||
pass
|
||||
|
||||
total_units = count_work_units(data)
|
||||
def _estimate_units(sysobj, fname) -> int:
|
||||
scopy = copy.deepcopy(sysobj)
|
||||
bar = _CountingBar()
|
||||
global PREFLIGHT_COUNT_MODE, PBAR
|
||||
saved_flag = PREFLIGHT_COUNT_MODE
|
||||
saved_pbar = PBAR
|
||||
PREFLIGHT_COUNT_MODE = True
|
||||
PBAR = bar
|
||||
try:
|
||||
try:
|
||||
searchSystem(scopy, bar)
|
||||
except Exception:
|
||||
pass
|
||||
return getattr(bar, "n", 0) or 0
|
||||
finally:
|
||||
PREFLIGHT_COUNT_MODE = saved_flag
|
||||
PBAR = saved_pbar
|
||||
|
||||
total_units = _estimate_units(data, filename)
|
||||
if not isinstance(total_units, int) or total_units <= 0:
|
||||
# Fallback: rough count
|
||||
total_units = 0
|
||||
for term in data.get("terms", {}) or {}:
|
||||
termList = data["terms"][term]
|
||||
if isinstance(termList, list):
|
||||
total_units += len(termList)
|
||||
gt = data.get("game_title")
|
||||
if isinstance(gt, str) and gt:
|
||||
total_units += 1
|
||||
total_units += len(data.get("variables", []) or [])
|
||||
total_units += len(data.get("weapon_types", []) or [])
|
||||
total_units += len(data.get("armor_types", []) or [])
|
||||
total_units += len(data.get("skill_types", []) or [])
|
||||
total_units += len(data.get("equip_types", []) or [])
|
||||
global PBAR
|
||||
|
||||
with tqdm(total=total_units, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar:
|
||||
|
|
@ -812,12 +1114,50 @@ def parseSystem(data, filename):
|
|||
|
||||
def parseScenario(data, filename):
|
||||
totalTokens = [0, 0]
|
||||
totalLines = 0
|
||||
global LOCK
|
||||
|
||||
# Get total for progress bar
|
||||
for page in data.items():
|
||||
totalLines += len(page[1])
|
||||
# --- Preflight: run searchCodes on each page list ---
|
||||
class _CountingBar:
|
||||
def __init__(self):
|
||||
self.n = 0
|
||||
def update(self, n=1):
|
||||
try:
|
||||
self.n += int(n) if n is not None else 1
|
||||
except Exception:
|
||||
self.n += 1
|
||||
def write(self, *args, **kwargs):
|
||||
pass
|
||||
def refresh(self):
|
||||
pass
|
||||
|
||||
def _estimate_units(scenario, fname) -> int:
|
||||
scopy = copy.deepcopy(scenario)
|
||||
bar = _CountingBar()
|
||||
global PREFLIGHT_COUNT_MODE, PBAR
|
||||
saved_flag = PREFLIGHT_COUNT_MODE
|
||||
saved_pbar = PBAR
|
||||
PREFLIGHT_COUNT_MODE = True
|
||||
PBAR = bar
|
||||
try:
|
||||
for key, lst in scopy.items():
|
||||
if lst is not None:
|
||||
try:
|
||||
searchCodes(lst, bar, [], fname)
|
||||
except Exception:
|
||||
pass
|
||||
return getattr(bar, "n", 0) or 0
|
||||
finally:
|
||||
PREFLIGHT_COUNT_MODE = saved_flag
|
||||
PBAR = saved_pbar
|
||||
|
||||
totalLines = _estimate_units(data, filename)
|
||||
if not isinstance(totalLines, int) or totalLines <= 0:
|
||||
totalLines = 0
|
||||
for _, lst in data.items():
|
||||
try:
|
||||
totalLines += len(lst or [])
|
||||
except Exception:
|
||||
pass
|
||||
global PBAR
|
||||
|
||||
with tqdm(total=totalLines, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar:
|
||||
|
|
@ -3394,6 +3734,21 @@ def translateAI(text, history, fullPromptFlag):
|
|||
# Return original text unmodified with zero tokens
|
||||
return [text, [0, 0]]
|
||||
|
||||
# Preflight count mode: don't hit API; just simulate progress units
|
||||
if 'PREFLIGHT_COUNT_MODE' in globals() and PREFLIGHT_COUNT_MODE:
|
||||
try:
|
||||
n = len(text) if isinstance(text, list) else 1
|
||||
except Exception:
|
||||
n = 1
|
||||
if PBAR is not None:
|
||||
try:
|
||||
with LOCK:
|
||||
PBAR.update(n)
|
||||
except Exception:
|
||||
pass
|
||||
# Return original payload and zero tokens so totals aren't affected
|
||||
return [text, [0, 0]]
|
||||
|
||||
return sharedtranslateAI(
|
||||
text=text,
|
||||
history=history,
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ PBAR = None
|
|||
FILENAME = None
|
||||
TIMETOTAL = 0 # Total Time Taken for all translations
|
||||
VOCAB_LOCK = threading.Lock()
|
||||
PREFLIGHT_COUNT_MODE = False # When True, translateAI wrapper only counts units and never calls API
|
||||
|
||||
# Speakers
|
||||
NAMESLIST = []
|
||||
|
|
@ -434,10 +435,107 @@ def update_vocab_section(category: str, pairs: list[tuple[str, str]]):
|
|||
|
||||
def parseMap(data, filename):
|
||||
totalTokens = [0, 0]
|
||||
totalLines = 0
|
||||
events = data["events"]
|
||||
global LOCK
|
||||
|
||||
# --- Preflight: estimate exact progress total using the same translation batching ---
|
||||
class _CountingBar:
|
||||
def __init__(self):
|
||||
self.n = 0
|
||||
def update(self, n=1):
|
||||
try:
|
||||
self.n += int(n) if n is not None else 1
|
||||
except Exception:
|
||||
self.n += 1
|
||||
def write(self, *args, **kwargs):
|
||||
# Silent during preflight
|
||||
pass
|
||||
def refresh(self):
|
||||
pass
|
||||
|
||||
def _estimate_map_units(d, fname) -> int:
|
||||
# Use a deep copy to avoid mutating real data during estimation
|
||||
dcopy = copy.deepcopy(d)
|
||||
bar = _CountingBar()
|
||||
# Temporarily enable preflight-count mode and route progress updates to our counter
|
||||
global PREFLIGHT_COUNT_MODE
|
||||
saved_preflight = PREFLIGHT_COUNT_MODE
|
||||
PREFLIGHT_COUNT_MODE = True
|
||||
global PBAR
|
||||
saved_pbar = PBAR
|
||||
PBAR = bar
|
||||
try:
|
||||
# Count display name TL (1 unit if present)
|
||||
if "Map" in fname and isinstance(dcopy.get("displayName", None), str):
|
||||
try:
|
||||
translateAI(
|
||||
dcopy["displayName"],
|
||||
"Reply with only the " + LANGUAGE + " translation of the RPG location name",
|
||||
False,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Notes and pages
|
||||
evts = dcopy.get("events", []) or []
|
||||
for evt in evts:
|
||||
if not evt:
|
||||
continue
|
||||
note_val = evt.get("note") or ""
|
||||
if not isinstance(note_val, str):
|
||||
note_val = str(note_val) if note_val is not None else ""
|
||||
|
||||
# <LB> name translation
|
||||
if "<LB>" in note_val:
|
||||
name_val = evt.get("name") or ""
|
||||
if isinstance(name_val, str) and name_val:
|
||||
try:
|
||||
translateAI(
|
||||
name_val,
|
||||
"Reply with only the " + LANGUAGE + " translation of the RPG location name",
|
||||
False,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# <msgText:"...">
|
||||
if "<msgText:" in note_val:
|
||||
try:
|
||||
translateNote(evt, r"<msgText:\"(.*?)\">", False)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# <namePop:...>, <LB:...>, <dn:...> handled before page processing in real run
|
||||
if "<namePop:" in note_val:
|
||||
try:
|
||||
translateNoteOmitSpace(evt, r"<namePop:\\s?([\\w一-龠ぁ-ゔァ-ヴーａ-ｚＡ-Ｚ０-９\\uFF61-\\uFF9F]+)")
|
||||
except Exception:
|
||||
pass
|
||||
if "<LB:" in note_val:
|
||||
try:
|
||||
translateNoteOmitSpace(evt, r"<LB:(.*?)\\s?>.*")
|
||||
except Exception:
|
||||
pass
|
||||
if "<dn:" in note_val:
|
||||
try:
|
||||
translateNoteOmitSpace(evt, r"<dn:\\s*(.*)>.*")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Other note tags handled during main loop
|
||||
# We'll just invoke the normal search pass which will batch and call translateAI
|
||||
for page in (evt.get("pages", []) or []):
|
||||
try:
|
||||
searchCodes(page, bar, [], fname)
|
||||
except Exception:
|
||||
# Ignore counting errors to avoid blocking
|
||||
pass
|
||||
|
||||
return getattr(bar, "n", 0) or 0
|
||||
finally:
|
||||
PREFLIGHT_COUNT_MODE = saved_preflight
|
||||
PBAR = saved_pbar
|
||||
|
||||
# Translate displayName for Map files
|
||||
if "Map" in filename:
|
||||
response = translateAI(
|
||||
|
|
@ -449,30 +547,18 @@ def parseMap(data, filename):
|
|||
totalTokens[1] += response[1][1]
|
||||
data["displayName"] = response[0].replace('"', "")
|
||||
|
||||
# Get total for progress bar (sum of all command list lengths across pages)
|
||||
for event in events:
|
||||
if event:
|
||||
note_val = event.get("note") or ""
|
||||
if not isinstance(note_val, str):
|
||||
note_val = str(note_val) if note_val is not None else ""
|
||||
if "<LB>" in note_val:
|
||||
# Translate event name when flagged with <LB>
|
||||
name_val = event.get("name") or ""
|
||||
if isinstance(name_val, str) and name_val:
|
||||
response = translateAI(
|
||||
name_val,
|
||||
"Reply with only the " + LANGUAGE + " translation of the RPG location name",
|
||||
False,
|
||||
)
|
||||
totalTokens[0] += response[1][0]
|
||||
totalTokens[1] += response[1][1]
|
||||
event["name"] = response[0].replace('"', "")
|
||||
if "<msgText:" in note_val:
|
||||
tokensResponse = translateNote(event, r"<msgText:\"(.*?)\">", False)
|
||||
totalTokens[0] += tokensResponse[0]
|
||||
totalTokens[1] += tokensResponse[1]
|
||||
for page in event["pages"]:
|
||||
totalLines += len(page["list"])
|
||||
# Compute accurate total using preflight (includes speakers, choices, groups, and notes)
|
||||
totalLines = _estimate_map_units(data, filename)
|
||||
if not isinstance(totalLines, int) or totalLines <= 0:
|
||||
# Fallback to naive count so a bar still renders
|
||||
totalLines = 0
|
||||
for event in events:
|
||||
if event:
|
||||
for page in event.get("pages", []) or []:
|
||||
try:
|
||||
totalLines += len(page.get("list", []))
|
||||
except Exception:
|
||||
pass
|
||||
global PBAR
|
||||
|
||||
# Process each page synchronously with progress updates
|
||||
|
|
@ -596,13 +682,52 @@ def translateNoteOmitSpace(event, regex):
|
|||
|
||||
def parseCommonEvents(data, filename):
|
||||
totalTokens = [0, 0]
|
||||
totalLines = 0
|
||||
global LOCK
|
||||
|
||||
# Get total for progress bar
|
||||
for page in data:
|
||||
if page is not None:
|
||||
totalLines += len(page["list"])
|
||||
# --- Preflight: estimate exact progress total using same batching ---
|
||||
class _CountingBar:
|
||||
def __init__(self):
|
||||
self.n = 0
|
||||
def update(self, n=1):
|
||||
try:
|
||||
self.n += int(n) if n is not None else 1
|
||||
except Exception:
|
||||
self.n += 1
|
||||
def write(self, *args, **kwargs):
|
||||
pass
|
||||
def refresh(self):
|
||||
pass
|
||||
|
||||
def _estimate_units(pages, fname) -> int:
|
||||
dcopy = copy.deepcopy(pages)
|
||||
bar = _CountingBar()
|
||||
global PREFLIGHT_COUNT_MODE, PBAR
|
||||
saved_flag = PREFLIGHT_COUNT_MODE
|
||||
saved_pbar = PBAR
|
||||
PREFLIGHT_COUNT_MODE = True
|
||||
PBAR = bar
|
||||
try:
|
||||
for page in dcopy:
|
||||
if page is not None:
|
||||
try:
|
||||
searchCodes(page, bar, [], fname)
|
||||
except Exception:
|
||||
pass
|
||||
return getattr(bar, "n", 0) or 0
|
||||
finally:
|
||||
PREFLIGHT_COUNT_MODE = saved_flag
|
||||
PBAR = saved_pbar
|
||||
|
||||
totalLines = _estimate_units(data, filename)
|
||||
if not isinstance(totalLines, int) or totalLines <= 0:
|
||||
# Fallback to naive command count
|
||||
totalLines = 0
|
||||
for page in data:
|
||||
if page is not None:
|
||||
try:
|
||||
totalLines += len(page.get("list", []))
|
||||
except Exception:
|
||||
pass
|
||||
global PBAR
|
||||
|
||||
with tqdm(total=totalLines, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar:
|
||||
|
|
@ -624,15 +749,55 @@ def parseCommonEvents(data, filename):
|
|||
|
||||
def parseTroops(data, filename):
|
||||
totalTokens = [0, 0]
|
||||
totalLines = 0
|
||||
global LOCK
|
||||
|
||||
# Get total for progress bar
|
||||
for troop in data:
|
||||
if troop is not None:
|
||||
for page in troop["pages"]:
|
||||
# Progress measured by number of commands in each page's list
|
||||
totalLines += len(page["list"])
|
||||
# --- Preflight total using same code paths ---
|
||||
class _CountingBar:
|
||||
def __init__(self):
|
||||
self.n = 0
|
||||
def update(self, n=1):
|
||||
try:
|
||||
self.n += int(n) if n is not None else 1
|
||||
except Exception:
|
||||
self.n += 1
|
||||
def write(self, *args, **kwargs):
|
||||
pass
|
||||
def refresh(self):
|
||||
pass
|
||||
|
||||
def _estimate_units(troops, fname) -> int:
|
||||
tcopy = copy.deepcopy(troops)
|
||||
bar = _CountingBar()
|
||||
global PREFLIGHT_COUNT_MODE, PBAR
|
||||
saved_flag = PREFLIGHT_COUNT_MODE
|
||||
saved_pbar = PBAR
|
||||
PREFLIGHT_COUNT_MODE = True
|
||||
PBAR = bar
|
||||
try:
|
||||
for troop in tcopy:
|
||||
if troop is None:
|
||||
continue
|
||||
for page in (troop.get("pages", []) or []):
|
||||
if page is not None:
|
||||
try:
|
||||
searchCodes(page, bar, [], fname)
|
||||
except Exception:
|
||||
pass
|
||||
return getattr(bar, "n", 0) or 0
|
||||
finally:
|
||||
PREFLIGHT_COUNT_MODE = saved_flag
|
||||
PBAR = saved_pbar
|
||||
|
||||
totalLines = _estimate_units(data, filename)
|
||||
if not isinstance(totalLines, int) or totalLines <= 0:
|
||||
totalLines = 0
|
||||
for troop in data:
|
||||
if troop is not None:
|
||||
for page in troop.get("pages", []) or []:
|
||||
try:
|
||||
totalLines += len(page.get("list", []))
|
||||
except Exception:
|
||||
pass
|
||||
global PBAR
|
||||
|
||||
with tqdm(total=totalLines, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar:
|
||||
|
|
@ -657,49 +822,138 @@ def parseTroops(data, filename):
|
|||
def parseNames(data, filename, context):
|
||||
totalTokens = [0, 0]
|
||||
|
||||
# Precompute total work units for progress bar
|
||||
def count_work_units(data, context):
|
||||
total = 0
|
||||
# --- Preflight: custom estimator that mirrors searchNames increments (incl. notes/messages) ---
|
||||
class _CountingBar:
|
||||
def __init__(self):
|
||||
self.n = 0
|
||||
def update(self, n=1):
|
||||
try:
|
||||
self.n += int(n) if n is not None else 1
|
||||
except Exception:
|
||||
self.n += 1
|
||||
def write(self, *args, **kwargs):
|
||||
pass
|
||||
def refresh(self):
|
||||
pass
|
||||
|
||||
def _estimate_names_units(entries, ctx, fname) -> int:
|
||||
ecopy = copy.deepcopy(entries)
|
||||
bar = _CountingBar()
|
||||
global PREFLIGHT_COUNT_MODE, PBAR
|
||||
saved_flag = PREFLIGHT_COUNT_MODE
|
||||
saved_pbar = PBAR
|
||||
PREFLIGHT_COUNT_MODE = True
|
||||
PBAR = bar
|
||||
try:
|
||||
# Count names/descriptions/profile/nickname
|
||||
name_cnt = 0
|
||||
desc_cnt = 0
|
||||
profile_cnt = 0
|
||||
nickname_cnt = 0
|
||||
msg_cnt = 0
|
||||
notes_cnt = 0
|
||||
|
||||
note_regexes = [
|
||||
(r"<note:(.*?)>", False),
|
||||
(r"<PE拡張:(.*?)>", False),
|
||||
(r"<hint:(.*?)>", False),
|
||||
(r"<SGDescription:(.*?)>", False),
|
||||
(r"<SG説明:\n?(.*?)>", False),
|
||||
(r"<SG説明2:\n?(.*?)>", False),
|
||||
(r"<SG説明3:\n?(.*?)>", False),
|
||||
(r"<SG説明4:\n?(.*?)>", False),
|
||||
(r"<SG説明:.+?Client\s?:.+?\n\n(.*?)>", True),
|
||||
(r"<SGカテゴリ:(.*?)>", False),
|
||||
(r"<Switch Shop Description>\n(.*)\n", False),
|
||||
(r"<MapText:(.*?)>", False),
|
||||
(r"WATs:(.+?)>", False),
|
||||
(r"ADTs?:(.+?)>", False),
|
||||
(r"<detail:(.*?)>", False),
|
||||
(r"<Name:(.*?)>", False),
|
||||
(r"<sub_1:([^>]+)", True),
|
||||
(r"<sub_2:([^>]+)", True),
|
||||
(r"<sub_3:([^>]+)", True),
|
||||
(r"<infowindow:(.*?)>", True),
|
||||
(r"<ExtendDesc:(.*?)>", True),
|
||||
(r"<desc\d:(.*?)>", False),
|
||||
(r"<拡張説明:(.+?)>", False),
|
||||
(r"<STS DESC>\n(.+?)\n<", False),
|
||||
(r"text:(.+)>", False),
|
||||
]
|
||||
|
||||
for entry in ecopy:
|
||||
if not entry:
|
||||
continue
|
||||
nm = entry.get("name") or ""
|
||||
ds = entry.get("description") or ""
|
||||
nn = entry.get("nickname") or ""
|
||||
pf = entry.get("profile") or ""
|
||||
if ctx == "Actors":
|
||||
if nm: name_cnt += 1
|
||||
if nn: nickname_cnt += 1
|
||||
if pf: profile_cnt += 1
|
||||
elif ctx in ["Armors", "Weapons", "Items"]:
|
||||
if nm: name_cnt += 1
|
||||
if ds: desc_cnt += 1
|
||||
elif ctx == "Skills":
|
||||
if nm: name_cnt += 1
|
||||
if ds: desc_cnt += 1
|
||||
for k in range(1,5):
|
||||
if entry.get(f"message{k}"): msg_cnt += 1
|
||||
elif ctx in ["Enemies", "Classes", "MapInfos"]:
|
||||
if nm: name_cnt += 1
|
||||
|
||||
# Notes counting
|
||||
note = entry.get("note") or ""
|
||||
if isinstance(note, str) and note:
|
||||
for regex, _ww in note_regexes:
|
||||
try:
|
||||
matches = re.findall(regex, note, re.DOTALL)
|
||||
except Exception:
|
||||
matches = []
|
||||
if regex.startswith(r"<SG説明:"):
|
||||
for m in matches:
|
||||
s = m if isinstance(m, str) else (m[0] if m else "")
|
||||
if "Client:" in s or "Client :" in s:
|
||||
continue
|
||||
notes_cnt += 1
|
||||
else:
|
||||
notes_cnt += len(matches)
|
||||
|
||||
# Simulate increments via wrapper (updates bar.n)
|
||||
if name_cnt:
|
||||
translateAI([""] * name_cnt, "", True)
|
||||
if desc_cnt:
|
||||
translateAI([""] * desc_cnt, "", True)
|
||||
if profile_cnt:
|
||||
translateAI([""] * profile_cnt, "", True)
|
||||
if nickname_cnt:
|
||||
translateAI([""] * nickname_cnt, "", True)
|
||||
if msg_cnt:
|
||||
translateAI([""] * msg_cnt, "", True)
|
||||
if notes_cnt:
|
||||
translateAI([""] * notes_cnt, "", True)
|
||||
|
||||
return getattr(bar, "n", 0) or 0
|
||||
finally:
|
||||
PREFLIGHT_COUNT_MODE = saved_flag
|
||||
PBAR = saved_pbar
|
||||
|
||||
total_units = _estimate_names_units(data, context, filename)
|
||||
if not isinstance(total_units, int) or total_units <= 0:
|
||||
# Reasonable fallback: count visible fields/messages (no notes)
|
||||
total_units = 0
|
||||
for entry in data:
|
||||
if not entry:
|
||||
continue
|
||||
|
||||
# Names and associated fields
|
||||
name = entry.get("name", "")
|
||||
desc = entry.get("description", "")
|
||||
nickname = entry.get("nickname", "")
|
||||
profile = entry.get("profile", "")
|
||||
|
||||
if entry.get("name"): total_units += 1
|
||||
if context in ["Armors", "Weapons", "Items", "Skills"] and entry.get("description"): total_units += 1
|
||||
if context == "Actors":
|
||||
if name:
|
||||
total += 1
|
||||
if nickname:
|
||||
total += 1
|
||||
if profile:
|
||||
total += 1
|
||||
elif context in ["Armors", "Weapons", "Items"]:
|
||||
if name:
|
||||
total += 1
|
||||
if desc:
|
||||
total += 1
|
||||
elif context == "Skills":
|
||||
if name:
|
||||
total += 1
|
||||
if desc:
|
||||
total += 1
|
||||
# Messages translated individually in searchNames
|
||||
for n in range(1, 5):
|
||||
msg = entry.get(f"message{n}")
|
||||
if msg:
|
||||
total += 1
|
||||
elif context in ["Enemies", "Classes", "MapInfos"]:
|
||||
if name:
|
||||
total += 1
|
||||
|
||||
return total
|
||||
|
||||
total_units = count_work_units(data, context)
|
||||
if entry.get("nickname"): total_units += 1
|
||||
if entry.get("profile"): total_units += 1
|
||||
if context == "Skills":
|
||||
for k in range(1,5):
|
||||
if entry.get(f"message{k}"): total_units += 1
|
||||
global PBAR
|
||||
|
||||
with tqdm(total=total_units, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar:
|
||||
|
|
@ -721,22 +975,50 @@ def parseNames(data, filename, context):
|
|||
def parseSS(data, filename):
|
||||
totalTokens = [0, 0]
|
||||
|
||||
# Precompute total units (ignore notes): name, description, message1..4 presence
|
||||
def count_work_units(states):
|
||||
total = 0
|
||||
for st in states:
|
||||
# --- Preflight using searchSS over deep copy ---
|
||||
class _CountingBar:
|
||||
def __init__(self):
|
||||
self.n = 0
|
||||
def update(self, n=1):
|
||||
try:
|
||||
self.n += int(n) if n is not None else 1
|
||||
except Exception:
|
||||
self.n += 1
|
||||
def write(self, *args, **kwargs):
|
||||
pass
|
||||
def refresh(self):
|
||||
pass
|
||||
|
||||
def _estimate_units(states, fname) -> int:
|
||||
scopy = copy.deepcopy(states)
|
||||
bar = _CountingBar()
|
||||
global PREFLIGHT_COUNT_MODE, PBAR
|
||||
saved_flag = PREFLIGHT_COUNT_MODE
|
||||
saved_pbar = PBAR
|
||||
PREFLIGHT_COUNT_MODE = True
|
||||
PBAR = bar
|
||||
try:
|
||||
for ss in scopy:
|
||||
if ss is not None:
|
||||
try:
|
||||
searchSS(ss, bar)
|
||||
except Exception:
|
||||
pass
|
||||
return getattr(bar, "n", 0) or 0
|
||||
finally:
|
||||
PREFLIGHT_COUNT_MODE = saved_flag
|
||||
PBAR = saved_pbar
|
||||
|
||||
total_units = _estimate_units(data, filename)
|
||||
if not isinstance(total_units, int) or total_units <= 0:
|
||||
total_units = 0
|
||||
for st in data:
|
||||
if not st:
|
||||
continue
|
||||
if st.get("name"):
|
||||
total += 1
|
||||
if st.get("description"):
|
||||
total += 1
|
||||
for n in range(1, 5):
|
||||
if st.get(f"message{n}"):
|
||||
total += 1
|
||||
return total
|
||||
|
||||
total_units = count_work_units(data)
|
||||
if st.get("name"): total_units += 1
|
||||
if st.get("description"): total_units += 1
|
||||
for n in range(1,5):
|
||||
if st.get(f"message{n}"): total_units += 1
|
||||
global PBAR
|
||||
|
||||
with tqdm(total=total_units, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar:
|
||||
|
|
@ -759,29 +1041,53 @@ def parseSS(data, filename):
|
|||
def parseSystem(data, filename):
|
||||
totalTokens = [0, 0]
|
||||
|
||||
# Precompute total units across system sections (exclude notes; count strings)
|
||||
def count_work_units(sys):
|
||||
total = 0
|
||||
# Title
|
||||
if sys.get("gameTitle"):
|
||||
total += 1
|
||||
# Terms (excluding 'messages' object)
|
||||
terms = sys.get("terms", {})
|
||||
for term_key, term_list in terms.items():
|
||||
if term_key == "messages":
|
||||
continue
|
||||
if isinstance(term_list, list):
|
||||
total += sum(1 for x in term_list if x is not None)
|
||||
# Armor, Skill, Equip types
|
||||
total += len(sys.get("armorTypes", []) or [])
|
||||
total += len(sys.get("skillTypes", []) or [])
|
||||
total += len(sys.get("equipTypes", []) or [])
|
||||
# Messages
|
||||
messages = terms.get("messages", {}) or {}
|
||||
total += len(messages)
|
||||
return total
|
||||
# --- Preflight: call searchSystem on deep copy to count increments ---
|
||||
class _CountingBar:
|
||||
def __init__(self):
|
||||
self.n = 0
|
||||
def update(self, n=1):
|
||||
try:
|
||||
self.n += int(n) if n is not None else 1
|
||||
except Exception:
|
||||
self.n += 1
|
||||
def write(self, *args, **kwargs):
|
||||
pass
|
||||
def refresh(self):
|
||||
pass
|
||||
|
||||
total_units = count_work_units(data)
|
||||
def _estimate_units(sysobj, fname) -> int:
|
||||
scopy = copy.deepcopy(sysobj)
|
||||
bar = _CountingBar()
|
||||
global PREFLIGHT_COUNT_MODE, PBAR
|
||||
saved_flag = PREFLIGHT_COUNT_MODE
|
||||
saved_pbar = PBAR
|
||||
PREFLIGHT_COUNT_MODE = True
|
||||
PBAR = bar
|
||||
try:
|
||||
try:
|
||||
searchSystem(scopy, bar)
|
||||
except Exception:
|
||||
pass
|
||||
return getattr(bar, "n", 0) or 0
|
||||
finally:
|
||||
PREFLIGHT_COUNT_MODE = saved_flag
|
||||
PBAR = saved_pbar
|
||||
|
||||
total_units = _estimate_units(data, filename)
|
||||
if not isinstance(total_units, int) or total_units <= 0:
|
||||
# Fallback: rough count of strings
|
||||
total_units = 0
|
||||
if data.get("gameTitle"): total_units += 1
|
||||
terms = data.get("terms", {}) or {}
|
||||
for k,v in terms.items():
|
||||
if k == "messages":
|
||||
continue
|
||||
if isinstance(v, list):
|
||||
total_units += sum(1 for x in v if x is not None)
|
||||
total_units += len(data.get("armorTypes", []) or [])
|
||||
total_units += len(data.get("skillTypes", []) or [])
|
||||
total_units += len(data.get("equipTypes", []) or [])
|
||||
total_units += len((terms.get("messages", {}) or {}))
|
||||
global PBAR
|
||||
|
||||
with tqdm(total=total_units, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar:
|
||||
|
|
@ -801,12 +1107,50 @@ def parseSystem(data, filename):
|
|||
|
||||
def parseScenario(data, filename):
|
||||
totalTokens = [0, 0]
|
||||
totalLines = 0
|
||||
global LOCK
|
||||
|
||||
# Get total for progress bar
|
||||
for page in data.items():
|
||||
totalLines += len(page[1])
|
||||
# --- Preflight: run searchCodes on each page list ---
|
||||
class _CountingBar:
|
||||
def __init__(self):
|
||||
self.n = 0
|
||||
def update(self, n=1):
|
||||
try:
|
||||
self.n += int(n) if n is not None else 1
|
||||
except Exception:
|
||||
self.n += 1
|
||||
def write(self, *args, **kwargs):
|
||||
pass
|
||||
def refresh(self):
|
||||
pass
|
||||
|
||||
def _estimate_units(scenario, fname) -> int:
|
||||
scopy = copy.deepcopy(scenario)
|
||||
bar = _CountingBar()
|
||||
global PREFLIGHT_COUNT_MODE, PBAR
|
||||
saved_flag = PREFLIGHT_COUNT_MODE
|
||||
saved_pbar = PBAR
|
||||
PREFLIGHT_COUNT_MODE = True
|
||||
PBAR = bar
|
||||
try:
|
||||
for key, lst in scopy.items():
|
||||
if lst is not None:
|
||||
try:
|
||||
searchCodes(lst, bar, [], fname)
|
||||
except Exception:
|
||||
pass
|
||||
return getattr(bar, "n", 0) or 0
|
||||
finally:
|
||||
PREFLIGHT_COUNT_MODE = saved_flag
|
||||
PBAR = saved_pbar
|
||||
|
||||
totalLines = _estimate_units(data, filename)
|
||||
if not isinstance(totalLines, int) or totalLines <= 0:
|
||||
totalLines = 0
|
||||
for _, lst in data.items():
|
||||
try:
|
||||
totalLines += len(lst or [])
|
||||
except Exception:
|
||||
pass
|
||||
global PBAR
|
||||
|
||||
with tqdm(total=totalLines, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar:
|
||||
|
|
@ -3187,6 +3531,21 @@ def translateAI(text, history, fullPromptFlag):
|
|||
# Return original text unmodified with zero tokens
|
||||
return [text, [0, 0]]
|
||||
|
||||
# Preflight count mode: don't hit API; just simulate progress units
|
||||
if 'PREFLIGHT_COUNT_MODE' in globals() and PREFLIGHT_COUNT_MODE:
|
||||
try:
|
||||
n = len(text) if isinstance(text, list) else 1
|
||||
except Exception:
|
||||
n = 1
|
||||
if PBAR is not None:
|
||||
try:
|
||||
with LOCK:
|
||||
PBAR.update(n)
|
||||
except Exception:
|
||||
pass
|
||||
# Return original payload and zero tokens so totals aren't affected
|
||||
return [text, [0, 0]]
|
||||
|
||||
return sharedtranslateAI(
|
||||
text=text,
|
||||
history=history,
|
||||
|
|
|
|||
Loading…
Reference in a new issue