From 5b6771d88c9393fe1b91b446a715e308446ac29d Mon Sep 17 00:00:00 2001 From: no Date: Sat, 1 Nov 2025 00:09:44 +0200 Subject: [PATCH] more accurate progress bar for mvmz + ace --- modules/rpgmakerace.py | 581 +++++++++++++++++++++++++++++++-------- modules/rpgmakermvmz.py | 589 ++++++++++++++++++++++++++++++++-------- 2 files changed, 942 insertions(+), 228 deletions(-) diff --git a/modules/rpgmakerace.py b/modules/rpgmakerace.py index a6e0281..4d54670 100644 --- a/modules/rpgmakerace.py +++ b/modules/rpgmakerace.py @@ -37,6 +37,7 @@ PBAR = None FILENAME = None TIMETOTAL = 0 # Total Time Taken for all translations VOCAB_LOCK = threading.Lock() +PREFLIGHT_COUNT_MODE = False # When True, translateAI wrapper only counts units and never calls API # Speakers NAMESLIST = [] @@ -447,10 +448,101 @@ def update_vocab_section(category: str, pairs: list[tuple[str, str]]): def parseMap(data, filename): totalTokens = [0, 0] - totalLines = 0 events = data["events"] global LOCK + # --- Preflight: estimate exact progress total using the same translation batching --- + class _CountingBar: + def __init__(self): + self.n = 0 + def update(self, n=1): + try: + self.n += int(n) if n is not None else 1 + except Exception: + self.n += 1 + def write(self, *args, **kwargs): + pass + def refresh(self): + pass + + def _estimate_map_units(d, fname) -> int: + dcopy = copy.deepcopy(d) + bar = _CountingBar() + global PREFLIGHT_COUNT_MODE + saved_preflight = PREFLIGHT_COUNT_MODE + global PBAR + saved_pbar = PBAR + PREFLIGHT_COUNT_MODE = True + PBAR = bar + try: + # Count display name TL (1 unit if present) + if "Map" in fname and isinstance(dcopy.get("display_name", None), str): + try: + translateAI( + dcopy["display_name"], + "Reply with only the " + LANGUAGE + " translation of the RPG location name", + False, + ) + except Exception: + pass + + # Notes and pages + evts = dcopy.get("events", {}) or {} + for evt in evts.values(): + if not evt: + continue + note_val = evt.get("note") or "" + if not isinstance(note_val, str): + note_val = str(note_val) if note_val is not None else "" + + # name translation + if "" in note_val: + name_val = evt.get("name") or "" + if isinstance(name_val, str) and name_val: + try: + translateAI( + name_val, + "Reply with only the " + LANGUAGE + " translation of the RPG location name", + False, + ) + except Exception: + pass + + # + if "", False) + except Exception: + pass + + # , , handled before page processing in real run + if ".*") + except Exception: + pass + if ".*") + except Exception: + pass + + for page in (evt.get("pages", []) or []): + try: + searchCodes(page, bar, [], fname) + except Exception: + pass + + return getattr(bar, "n", 0) or 0 + finally: + PREFLIGHT_COUNT_MODE = saved_preflight + PBAR = saved_pbar + # Translate display_name for Map files if "Map" in filename: response = translateAI( @@ -462,30 +554,17 @@ def parseMap(data, filename): totalTokens[1] += response[1][1] data["display_name"] = response[0].replace('"', "") - # Get total for progress bar (sum of all command list lengths across pages) - for event in events.values(): - if event and isinstance(event, dict): - note_val = event.get("note") or "" - if not isinstance(note_val, str): - note_val = str(note_val) if note_val is not None else "" - if "" in note_val: - # Translate event name when flagged with - name_val = event.get("name") or "" - if isinstance(name_val, str) and name_val: - response = translateAI( - name_val, - "Reply with only the " + LANGUAGE + " translation of the RPG location name", - False, - ) - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - event["name"] = response[0].replace('"', "") - if "", False) - totalTokens[0] += tokensResponse[0] - totalTokens[1] += tokensResponse[1] - for page in event["pages"]: - totalLines += len(page["list"]) + totalLines = _estimate_map_units(data, filename) + if not isinstance(totalLines, int) or totalLines <= 0: + # Fallback to naive count so a bar still renders + totalLines = 0 + for evt in events.values(): + if evt: + for page in (evt.get("pages", []) or []): + try: + totalLines += len(page.get("list", [])) + except Exception: + pass global PBAR # Process each page synchronously with progress updates @@ -609,13 +688,52 @@ def translateNoteOmitSpace(event, regex): def parseCommonEvents(data, filename): totalTokens = [0, 0] - totalLines = 0 global LOCK - # Get total for progress bar - for page in data: - if page is not None: - totalLines += len(page["list"]) + # --- Preflight: estimate exact progress total using same batching --- + class _CountingBar: + def __init__(self): + self.n = 0 + def update(self, n=1): + try: + self.n += int(n) if n is not None else 1 + except Exception: + self.n += 1 + def write(self, *args, **kwargs): + pass + def refresh(self): + pass + + def _estimate_units(pages, fname) -> int: + dcopy = copy.deepcopy(pages) + bar = _CountingBar() + global PREFLIGHT_COUNT_MODE, PBAR + saved_flag = PREFLIGHT_COUNT_MODE + saved_pbar = PBAR + PREFLIGHT_COUNT_MODE = True + PBAR = bar + try: + for page in dcopy: + if page is not None: + try: + searchCodes(page, bar, [], fname) + except Exception: + pass + return getattr(bar, "n", 0) or 0 + finally: + PREFLIGHT_COUNT_MODE = saved_flag + PBAR = saved_pbar + + totalLines = _estimate_units(data, filename) + if not isinstance(totalLines, int) or totalLines <= 0: + # Fallback to naive command count + totalLines = 0 + for page in data: + if page is not None: + try: + totalLines += len(page.get("list", [])) + except Exception: + pass global PBAR with tqdm(total=totalLines, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar: @@ -637,15 +755,55 @@ def parseCommonEvents(data, filename): def parseTroops(data, filename): totalTokens = [0, 0] - totalLines = 0 global LOCK - # Get total for progress bar - for troop in data: - if troop is not None: - for page in troop["pages"]: - # Progress measured by number of commands in each page's list - totalLines += len(page["list"]) + # --- Preflight total using same code paths --- + class _CountingBar: + def __init__(self): + self.n = 0 + def update(self, n=1): + try: + self.n += int(n) if n is not None else 1 + except Exception: + self.n += 1 + def write(self, *args, **kwargs): + pass + def refresh(self): + pass + + def _estimate_units(troops, fname) -> int: + tcopy = copy.deepcopy(troops) + bar = _CountingBar() + global PREFLIGHT_COUNT_MODE, PBAR + saved_flag = PREFLIGHT_COUNT_MODE + saved_pbar = PBAR + PREFLIGHT_COUNT_MODE = True + PBAR = bar + try: + for troop in tcopy: + if troop is None: + continue + for page in (troop.get("pages", []) or []): + if page is not None: + try: + searchCodes(page, bar, [], fname) + except Exception: + pass + return getattr(bar, "n", 0) or 0 + finally: + PREFLIGHT_COUNT_MODE = saved_flag + PBAR = saved_pbar + + totalLines = _estimate_units(data, filename) + if not isinstance(totalLines, int) or totalLines <= 0: + totalLines = 0 + for troop in data: + if troop is not None: + for page in troop.get("pages", []) or []: + try: + totalLines += len(page.get("list", [])) + except Exception: + pass global PBAR with tqdm(total=totalLines, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar: @@ -670,49 +828,138 @@ def parseTroops(data, filename): def parseNames(data, filename, context): totalTokens = [0, 0] - # Precompute total work units for progress bar - def count_work_units(data, context): - total = 0 + # --- Preflight: custom estimator that mirrors searchNames increments (incl. notes/messages) --- + class _CountingBar: + def __init__(self): + self.n = 0 + def update(self, n=1): + try: + self.n += int(n) if n is not None else 1 + except Exception: + self.n += 1 + def write(self, *args, **kwargs): + pass + def refresh(self): + pass + def _estimate_names_units(entries, ctx, fname) -> int: + ecopy = copy.deepcopy(entries) + bar = _CountingBar() + global PREFLIGHT_COUNT_MODE, PBAR + saved_flag = PREFLIGHT_COUNT_MODE + saved_pbar = PBAR + PREFLIGHT_COUNT_MODE = True + PBAR = bar + try: + # Counts + name_cnt = 0 + desc_cnt = 0 + profile_cnt = 0 + nickname_cnt = 0 + msg_cnt = 0 + notes_cnt = 0 + + note_regexes = [ + (r"", False), + (r"", False), + (r"", False), + (r"", False), + (r"", False), + (r"", False), + (r"", False), + (r"", False), + (r"", True), + (r"", False), + (r"\n(.*)\n", False), + (r"", False), + (r"WATs:(.+?)>", False), + (r"ADTs?:(.+?)>", False), + (r"", False), + (r"", False), + (r"]+)", True), + (r"]+)", True), + (r"]+)", True), + (r"", True), + (r"", True), + (r"", False), + (r"<拡張説明:(.+?)>", False), + (r"\n(.+?)\n<", False), + (r"text:(.+)>", False), + ] + + for entry in ecopy: + if not entry: + continue + nm = entry.get("name") or "" + ds = entry.get("description") or "" + nn = entry.get("nickname") or "" + pf = entry.get("profile") or "" + if ctx == "Actors": + if nm: name_cnt += 1 + if nn: nickname_cnt += 1 + if pf: profile_cnt += 1 + elif ctx in ["Armors", "Weapons", "Items"]: + if nm: name_cnt += 1 + if ds: desc_cnt += 1 + elif ctx == "Skills": + if nm: name_cnt += 1 + if ds: desc_cnt += 1 + for k in range(1,5): + if entry.get(f"message{k}"): msg_cnt += 1 + elif ctx in ["Enemies", "Classes", "MapInfos"]: + if nm: name_cnt += 1 + + # Notes counting + note = entry.get("note") or "" + if isinstance(note, str) and note: + for regex, _ww in note_regexes: + try: + matches = re.findall(regex, note, re.DOTALL) + except Exception: + matches = [] + if regex.startswith(r" int: + scopy = copy.deepcopy(states) + bar = _CountingBar() + global PREFLIGHT_COUNT_MODE, PBAR + saved_flag = PREFLIGHT_COUNT_MODE + saved_pbar = PBAR + PREFLIGHT_COUNT_MODE = True + PBAR = bar + try: + for ss in scopy: + if ss is not None: + try: + searchSS(ss, bar) + except Exception: + pass + return getattr(bar, "n", 0) or 0 + finally: + PREFLIGHT_COUNT_MODE = saved_flag + PBAR = saved_pbar + + total_units = _estimate_units(data, filename) + if not isinstance(total_units, int) or total_units <= 0: + total_units = 0 + for st in data: if not st: continue - if st.get("name"): - total += 1 - if st.get("description"): - total += 1 - for n in range(1, 5): - if st.get(f"message{n}"): - total += 1 - return total - - total_units = count_work_units(data) + if st.get("name"): total_units += 1 + if st.get("description"): total_units += 1 + for n in range(1,5): + if st.get(f"message{n}"): total_units += 1 global PBAR with tqdm(total=total_units, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar: @@ -772,27 +1047,54 @@ def parseSS(data, filename): def parseSystem(data, filename): totalTokens = [0, 0] - # Precompute total units for Ace schema - def count_work_units(sys): - total = 0 - # Terms: sum list lengths of each term list - for term in sys.get("terms", {}): - termList = sys["terms"][term] - if isinstance(termList, list): - total += len(termList) - # game_title might be a string; count as 1 if non-empty - gt = sys.get("game_title") - if isinstance(gt, str) and gt: - total += 1 - # variables is a list - total += len(sys.get("variables", []) or []) - total += len(sys.get("weapon_types", []) or []) - total += len(sys.get("armor_types", []) or []) - total += len(sys.get("skill_types", []) or []) - total += len(sys.get("equip_types", []) or []) - return total + # --- Preflight: call searchSystem on deep copy to count increments --- + class _CountingBar: + def __init__(self): + self.n = 0 + def update(self, n=1): + try: + self.n += int(n) if n is not None else 1 + except Exception: + self.n += 1 + def write(self, *args, **kwargs): + pass + def refresh(self): + pass - total_units = count_work_units(data) + def _estimate_units(sysobj, fname) -> int: + scopy = copy.deepcopy(sysobj) + bar = _CountingBar() + global PREFLIGHT_COUNT_MODE, PBAR + saved_flag = PREFLIGHT_COUNT_MODE + saved_pbar = PBAR + PREFLIGHT_COUNT_MODE = True + PBAR = bar + try: + try: + searchSystem(scopy, bar) + except Exception: + pass + return getattr(bar, "n", 0) or 0 + finally: + PREFLIGHT_COUNT_MODE = saved_flag + PBAR = saved_pbar + + total_units = _estimate_units(data, filename) + if not isinstance(total_units, int) or total_units <= 0: + # Fallback: rough count + total_units = 0 + for term in data.get("terms", {}) or {}: + termList = data["terms"][term] + if isinstance(termList, list): + total_units += len(termList) + gt = data.get("game_title") + if isinstance(gt, str) and gt: + total_units += 1 + total_units += len(data.get("variables", []) or []) + total_units += len(data.get("weapon_types", []) or []) + total_units += len(data.get("armor_types", []) or []) + total_units += len(data.get("skill_types", []) or []) + total_units += len(data.get("equip_types", []) or []) global PBAR with tqdm(total=total_units, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar: @@ -812,12 +1114,50 @@ def parseSystem(data, filename): def parseScenario(data, filename): totalTokens = [0, 0] - totalLines = 0 global LOCK - # Get total for progress bar - for page in data.items(): - totalLines += len(page[1]) + # --- Preflight: run searchCodes on each page list --- + class _CountingBar: + def __init__(self): + self.n = 0 + def update(self, n=1): + try: + self.n += int(n) if n is not None else 1 + except Exception: + self.n += 1 + def write(self, *args, **kwargs): + pass + def refresh(self): + pass + + def _estimate_units(scenario, fname) -> int: + scopy = copy.deepcopy(scenario) + bar = _CountingBar() + global PREFLIGHT_COUNT_MODE, PBAR + saved_flag = PREFLIGHT_COUNT_MODE + saved_pbar = PBAR + PREFLIGHT_COUNT_MODE = True + PBAR = bar + try: + for key, lst in scopy.items(): + if lst is not None: + try: + searchCodes(lst, bar, [], fname) + except Exception: + pass + return getattr(bar, "n", 0) or 0 + finally: + PREFLIGHT_COUNT_MODE = saved_flag + PBAR = saved_pbar + + totalLines = _estimate_units(data, filename) + if not isinstance(totalLines, int) or totalLines <= 0: + totalLines = 0 + for _, lst in data.items(): + try: + totalLines += len(lst or []) + except Exception: + pass global PBAR with tqdm(total=totalLines, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar: @@ -3394,6 +3734,21 @@ def translateAI(text, history, fullPromptFlag): # Return original text unmodified with zero tokens return [text, [0, 0]] + # Preflight count mode: don't hit API; just simulate progress units + if 'PREFLIGHT_COUNT_MODE' in globals() and PREFLIGHT_COUNT_MODE: + try: + n = len(text) if isinstance(text, list) else 1 + except Exception: + n = 1 + if PBAR is not None: + try: + with LOCK: + PBAR.update(n) + except Exception: + pass + # Return original payload and zero tokens so totals aren't affected + return [text, [0, 0]] + return sharedtranslateAI( text=text, history=history, diff --git a/modules/rpgmakermvmz.py b/modules/rpgmakermvmz.py index 3951b20..1b26e46 100644 --- a/modules/rpgmakermvmz.py +++ b/modules/rpgmakermvmz.py @@ -35,6 +35,7 @@ PBAR = None FILENAME = None TIMETOTAL = 0 # Total Time Taken for all translations VOCAB_LOCK = threading.Lock() +PREFLIGHT_COUNT_MODE = False # When True, translateAI wrapper only counts units and never calls API # Speakers NAMESLIST = [] @@ -434,10 +435,107 @@ def update_vocab_section(category: str, pairs: list[tuple[str, str]]): def parseMap(data, filename): totalTokens = [0, 0] - totalLines = 0 events = data["events"] global LOCK + # --- Preflight: estimate exact progress total using the same translation batching --- + class _CountingBar: + def __init__(self): + self.n = 0 + def update(self, n=1): + try: + self.n += int(n) if n is not None else 1 + except Exception: + self.n += 1 + def write(self, *args, **kwargs): + # Silent during preflight + pass + def refresh(self): + pass + + def _estimate_map_units(d, fname) -> int: + # Use a deep copy to avoid mutating real data during estimation + dcopy = copy.deepcopy(d) + bar = _CountingBar() + # Temporarily enable preflight-count mode and route progress updates to our counter + global PREFLIGHT_COUNT_MODE + saved_preflight = PREFLIGHT_COUNT_MODE + PREFLIGHT_COUNT_MODE = True + global PBAR + saved_pbar = PBAR + PBAR = bar + try: + # Count display name TL (1 unit if present) + if "Map" in fname and isinstance(dcopy.get("displayName", None), str): + try: + translateAI( + dcopy["displayName"], + "Reply with only the " + LANGUAGE + " translation of the RPG location name", + False, + ) + except Exception: + pass + + # Notes and pages + evts = dcopy.get("events", []) or [] + for evt in evts: + if not evt: + continue + note_val = evt.get("note") or "" + if not isinstance(note_val, str): + note_val = str(note_val) if note_val is not None else "" + + # name translation + if "" in note_val: + name_val = evt.get("name") or "" + if isinstance(name_val, str) and name_val: + try: + translateAI( + name_val, + "Reply with only the " + LANGUAGE + " translation of the RPG location name", + False, + ) + except Exception: + pass + + # + if "", False) + except Exception: + pass + + # , , handled before page processing in real run + if ".*") + except Exception: + pass + if ".*") + except Exception: + pass + + # Other note tags handled during main loop + # We'll just invoke the normal search pass which will batch and call translateAI + for page in (evt.get("pages", []) or []): + try: + searchCodes(page, bar, [], fname) + except Exception: + # Ignore counting errors to avoid blocking + pass + + return getattr(bar, "n", 0) or 0 + finally: + PREFLIGHT_COUNT_MODE = saved_preflight + PBAR = saved_pbar + # Translate displayName for Map files if "Map" in filename: response = translateAI( @@ -449,30 +547,18 @@ def parseMap(data, filename): totalTokens[1] += response[1][1] data["displayName"] = response[0].replace('"', "") - # Get total for progress bar (sum of all command list lengths across pages) - for event in events: - if event: - note_val = event.get("note") or "" - if not isinstance(note_val, str): - note_val = str(note_val) if note_val is not None else "" - if "" in note_val: - # Translate event name when flagged with - name_val = event.get("name") or "" - if isinstance(name_val, str) and name_val: - response = translateAI( - name_val, - "Reply with only the " + LANGUAGE + " translation of the RPG location name", - False, - ) - totalTokens[0] += response[1][0] - totalTokens[1] += response[1][1] - event["name"] = response[0].replace('"', "") - if "", False) - totalTokens[0] += tokensResponse[0] - totalTokens[1] += tokensResponse[1] - for page in event["pages"]: - totalLines += len(page["list"]) + # Compute accurate total using preflight (includes speakers, choices, groups, and notes) + totalLines = _estimate_map_units(data, filename) + if not isinstance(totalLines, int) or totalLines <= 0: + # Fallback to naive count so a bar still renders + totalLines = 0 + for event in events: + if event: + for page in event.get("pages", []) or []: + try: + totalLines += len(page.get("list", [])) + except Exception: + pass global PBAR # Process each page synchronously with progress updates @@ -596,13 +682,52 @@ def translateNoteOmitSpace(event, regex): def parseCommonEvents(data, filename): totalTokens = [0, 0] - totalLines = 0 global LOCK - # Get total for progress bar - for page in data: - if page is not None: - totalLines += len(page["list"]) + # --- Preflight: estimate exact progress total using same batching --- + class _CountingBar: + def __init__(self): + self.n = 0 + def update(self, n=1): + try: + self.n += int(n) if n is not None else 1 + except Exception: + self.n += 1 + def write(self, *args, **kwargs): + pass + def refresh(self): + pass + + def _estimate_units(pages, fname) -> int: + dcopy = copy.deepcopy(pages) + bar = _CountingBar() + global PREFLIGHT_COUNT_MODE, PBAR + saved_flag = PREFLIGHT_COUNT_MODE + saved_pbar = PBAR + PREFLIGHT_COUNT_MODE = True + PBAR = bar + try: + for page in dcopy: + if page is not None: + try: + searchCodes(page, bar, [], fname) + except Exception: + pass + return getattr(bar, "n", 0) or 0 + finally: + PREFLIGHT_COUNT_MODE = saved_flag + PBAR = saved_pbar + + totalLines = _estimate_units(data, filename) + if not isinstance(totalLines, int) or totalLines <= 0: + # Fallback to naive command count + totalLines = 0 + for page in data: + if page is not None: + try: + totalLines += len(page.get("list", [])) + except Exception: + pass global PBAR with tqdm(total=totalLines, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar: @@ -624,15 +749,55 @@ def parseCommonEvents(data, filename): def parseTroops(data, filename): totalTokens = [0, 0] - totalLines = 0 global LOCK - # Get total for progress bar - for troop in data: - if troop is not None: - for page in troop["pages"]: - # Progress measured by number of commands in each page's list - totalLines += len(page["list"]) + # --- Preflight total using same code paths --- + class _CountingBar: + def __init__(self): + self.n = 0 + def update(self, n=1): + try: + self.n += int(n) if n is not None else 1 + except Exception: + self.n += 1 + def write(self, *args, **kwargs): + pass + def refresh(self): + pass + + def _estimate_units(troops, fname) -> int: + tcopy = copy.deepcopy(troops) + bar = _CountingBar() + global PREFLIGHT_COUNT_MODE, PBAR + saved_flag = PREFLIGHT_COUNT_MODE + saved_pbar = PBAR + PREFLIGHT_COUNT_MODE = True + PBAR = bar + try: + for troop in tcopy: + if troop is None: + continue + for page in (troop.get("pages", []) or []): + if page is not None: + try: + searchCodes(page, bar, [], fname) + except Exception: + pass + return getattr(bar, "n", 0) or 0 + finally: + PREFLIGHT_COUNT_MODE = saved_flag + PBAR = saved_pbar + + totalLines = _estimate_units(data, filename) + if not isinstance(totalLines, int) or totalLines <= 0: + totalLines = 0 + for troop in data: + if troop is not None: + for page in troop.get("pages", []) or []: + try: + totalLines += len(page.get("list", [])) + except Exception: + pass global PBAR with tqdm(total=totalLines, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar: @@ -657,49 +822,138 @@ def parseTroops(data, filename): def parseNames(data, filename, context): totalTokens = [0, 0] - # Precompute total work units for progress bar - def count_work_units(data, context): - total = 0 + # --- Preflight: custom estimator that mirrors searchNames increments (incl. notes/messages) --- + class _CountingBar: + def __init__(self): + self.n = 0 + def update(self, n=1): + try: + self.n += int(n) if n is not None else 1 + except Exception: + self.n += 1 + def write(self, *args, **kwargs): + pass + def refresh(self): + pass + def _estimate_names_units(entries, ctx, fname) -> int: + ecopy = copy.deepcopy(entries) + bar = _CountingBar() + global PREFLIGHT_COUNT_MODE, PBAR + saved_flag = PREFLIGHT_COUNT_MODE + saved_pbar = PBAR + PREFLIGHT_COUNT_MODE = True + PBAR = bar + try: + # Count names/descriptions/profile/nickname + name_cnt = 0 + desc_cnt = 0 + profile_cnt = 0 + nickname_cnt = 0 + msg_cnt = 0 + notes_cnt = 0 + + note_regexes = [ + (r"", False), + (r"", False), + (r"", False), + (r"", False), + (r"", False), + (r"", False), + (r"", False), + (r"", False), + (r"", True), + (r"", False), + (r"\n(.*)\n", False), + (r"", False), + (r"WATs:(.+?)>", False), + (r"ADTs?:(.+?)>", False), + (r"", False), + (r"", False), + (r"]+)", True), + (r"]+)", True), + (r"]+)", True), + (r"", True), + (r"", True), + (r"", False), + (r"<拡張説明:(.+?)>", False), + (r"\n(.+?)\n<", False), + (r"text:(.+)>", False), + ] + + for entry in ecopy: + if not entry: + continue + nm = entry.get("name") or "" + ds = entry.get("description") or "" + nn = entry.get("nickname") or "" + pf = entry.get("profile") or "" + if ctx == "Actors": + if nm: name_cnt += 1 + if nn: nickname_cnt += 1 + if pf: profile_cnt += 1 + elif ctx in ["Armors", "Weapons", "Items"]: + if nm: name_cnt += 1 + if ds: desc_cnt += 1 + elif ctx == "Skills": + if nm: name_cnt += 1 + if ds: desc_cnt += 1 + for k in range(1,5): + if entry.get(f"message{k}"): msg_cnt += 1 + elif ctx in ["Enemies", "Classes", "MapInfos"]: + if nm: name_cnt += 1 + + # Notes counting + note = entry.get("note") or "" + if isinstance(note, str) and note: + for regex, _ww in note_regexes: + try: + matches = re.findall(regex, note, re.DOTALL) + except Exception: + matches = [] + if regex.startswith(r" int: + scopy = copy.deepcopy(states) + bar = _CountingBar() + global PREFLIGHT_COUNT_MODE, PBAR + saved_flag = PREFLIGHT_COUNT_MODE + saved_pbar = PBAR + PREFLIGHT_COUNT_MODE = True + PBAR = bar + try: + for ss in scopy: + if ss is not None: + try: + searchSS(ss, bar) + except Exception: + pass + return getattr(bar, "n", 0) or 0 + finally: + PREFLIGHT_COUNT_MODE = saved_flag + PBAR = saved_pbar + + total_units = _estimate_units(data, filename) + if not isinstance(total_units, int) or total_units <= 0: + total_units = 0 + for st in data: if not st: continue - if st.get("name"): - total += 1 - if st.get("description"): - total += 1 - for n in range(1, 5): - if st.get(f"message{n}"): - total += 1 - return total - - total_units = count_work_units(data) + if st.get("name"): total_units += 1 + if st.get("description"): total_units += 1 + for n in range(1,5): + if st.get(f"message{n}"): total_units += 1 global PBAR with tqdm(total=total_units, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar: @@ -759,29 +1041,53 @@ def parseSS(data, filename): def parseSystem(data, filename): totalTokens = [0, 0] - # Precompute total units across system sections (exclude notes; count strings) - def count_work_units(sys): - total = 0 - # Title - if sys.get("gameTitle"): - total += 1 - # Terms (excluding 'messages' object) - terms = sys.get("terms", {}) - for term_key, term_list in terms.items(): - if term_key == "messages": - continue - if isinstance(term_list, list): - total += sum(1 for x in term_list if x is not None) - # Armor, Skill, Equip types - total += len(sys.get("armorTypes", []) or []) - total += len(sys.get("skillTypes", []) or []) - total += len(sys.get("equipTypes", []) or []) - # Messages - messages = terms.get("messages", {}) or {} - total += len(messages) - return total + # --- Preflight: call searchSystem on deep copy to count increments --- + class _CountingBar: + def __init__(self): + self.n = 0 + def update(self, n=1): + try: + self.n += int(n) if n is not None else 1 + except Exception: + self.n += 1 + def write(self, *args, **kwargs): + pass + def refresh(self): + pass - total_units = count_work_units(data) + def _estimate_units(sysobj, fname) -> int: + scopy = copy.deepcopy(sysobj) + bar = _CountingBar() + global PREFLIGHT_COUNT_MODE, PBAR + saved_flag = PREFLIGHT_COUNT_MODE + saved_pbar = PBAR + PREFLIGHT_COUNT_MODE = True + PBAR = bar + try: + try: + searchSystem(scopy, bar) + except Exception: + pass + return getattr(bar, "n", 0) or 0 + finally: + PREFLIGHT_COUNT_MODE = saved_flag + PBAR = saved_pbar + + total_units = _estimate_units(data, filename) + if not isinstance(total_units, int) or total_units <= 0: + # Fallback: rough count of strings + total_units = 0 + if data.get("gameTitle"): total_units += 1 + terms = data.get("terms", {}) or {} + for k,v in terms.items(): + if k == "messages": + continue + if isinstance(v, list): + total_units += sum(1 for x in v if x is not None) + total_units += len(data.get("armorTypes", []) or []) + total_units += len(data.get("skillTypes", []) or []) + total_units += len(data.get("equipTypes", []) or []) + total_units += len((terms.get("messages", {}) or {})) global PBAR with tqdm(total=total_units, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar: @@ -801,12 +1107,50 @@ def parseSystem(data, filename): def parseScenario(data, filename): totalTokens = [0, 0] - totalLines = 0 global LOCK - # Get total for progress bar - for page in data.items(): - totalLines += len(page[1]) + # --- Preflight: run searchCodes on each page list --- + class _CountingBar: + def __init__(self): + self.n = 0 + def update(self, n=1): + try: + self.n += int(n) if n is not None else 1 + except Exception: + self.n += 1 + def write(self, *args, **kwargs): + pass + def refresh(self): + pass + + def _estimate_units(scenario, fname) -> int: + scopy = copy.deepcopy(scenario) + bar = _CountingBar() + global PREFLIGHT_COUNT_MODE, PBAR + saved_flag = PREFLIGHT_COUNT_MODE + saved_pbar = PBAR + PREFLIGHT_COUNT_MODE = True + PBAR = bar + try: + for key, lst in scopy.items(): + if lst is not None: + try: + searchCodes(lst, bar, [], fname) + except Exception: + pass + return getattr(bar, "n", 0) or 0 + finally: + PREFLIGHT_COUNT_MODE = saved_flag + PBAR = saved_pbar + + totalLines = _estimate_units(data, filename) + if not isinstance(totalLines, int) or totalLines <= 0: + totalLines = 0 + for _, lst in data.items(): + try: + totalLines += len(lst or []) + except Exception: + pass global PBAR with tqdm(total=totalLines, bar_format=BAR_FORMAT, position=POSITION, leave=LEAVE, desc=filename) as pbar: @@ -3187,6 +3531,21 @@ def translateAI(text, history, fullPromptFlag): # Return original text unmodified with zero tokens return [text, [0, 0]] + # Preflight count mode: don't hit API; just simulate progress units + if 'PREFLIGHT_COUNT_MODE' in globals() and PREFLIGHT_COUNT_MODE: + try: + n = len(text) if isinstance(text, list) else 1 + except Exception: + n = 1 + if PBAR is not None: + try: + with LOCK: + PBAR.update(n) + except Exception: + pass + # Return original payload and zero tokens so totals aren't affected + return [text, [0, 0]] + return sharedtranslateAI( text=text, history=history,