diff --git a/gui/wolf_workflow_tab.py b/gui/wolf_workflow_tab.py index df1c719..4b21989 100644 --- a/gui/wolf_workflow_tab.py +++ b/gui/wolf_workflow_tab.py @@ -92,6 +92,7 @@ from util.paths import PROJECT_ROOT from util.project_scanner import ( detect_wolf_layout, find_wolf_text_archives, + find_wolf_unpack_gaps, list_wolf_json_files, wolf_has_maps, wolf_maps_dir, @@ -1077,6 +1078,19 @@ class WolfWorkflowTab(QWidget): if self._worker is not None and self._worker.isRunning(): return + info = self._layout or detect_wolf_layout(self._game_root) + self._layout = info + gaps = info.get("unpack_gaps") or [] + if gaps or (info.get("archives") and not info.get("unpacked")): + pending = gaps or info.get("archives") or [] + self._log(f"Auto-setup: unpacking {len(pending)} archive(s) …") + self._unpack( + on_done=self._on_auto_unpack_done, + interactive=False, + archives=pending, + ) + return + manifest = self._read_manifest() if manifest and manifest.get("entries"): info = self._layout or detect_wolf_layout(self._game_root) @@ -1102,12 +1116,6 @@ class WolfWorkflowTab(QWidget): self._scan_wolf_files() return - info = self._layout or detect_wolf_layout(self._game_root) - self._layout = info - if info.get("archives") and not info.get("unpacked"): - self._log("Auto-setup: unpacking .wolf archives …") - self._unpack(on_done=self._on_auto_unpack_done, interactive=False) - return if info.get("data_dir"): self._log("Auto-setup: extracting text into wolf_json/ …") self._extract_to_work_dir(on_done=lambda ok, _m: self._scan_wolf_files() if ok else None) @@ -1118,6 +1126,11 @@ class WolfWorkflowTab(QWidget): if not ok: return self._layout = detect_wolf_layout(self._game_root) + gaps = self._layout.get("unpack_gaps") or [] + if gaps: + names = ", ".join(a.name for a in gaps) + self._log(f"⚠ Unpack still incomplete: {names}") + return if self._layout.get("data_dir"): self._log("Auto-setup: extracting text into wolf_json/ …") self._extract_to_work_dir(on_done=lambda ok2, _m: self._scan_wolf_files() if ok2 else None) @@ -1401,12 +1414,14 @@ class WolfWorkflowTab(QWidget): self._run_task(task) - def _unpack(self, on_done=None, *, interactive: bool = True): + def _unpack(self, on_done=None, *, interactive: bool = True, archives=None): if not self._require_root(): return root = Path(self._game_root) info = detect_wolf_layout(root) - archives = info["archives"] + if archives is None: + gaps = info.get("unpack_gaps") or find_wolf_unpack_gaps(root) + archives = gaps if gaps else info["archives"] if not archives: self._log("Unpack: no .wolf archives found (already unpacked?).") if interactive: diff --git a/tests/test_wolf_project_scanner.py b/tests/test_wolf_project_scanner.py index e68bdb6..4290682 100644 --- a/tests/test_wolf_project_scanner.py +++ b/tests/test_wolf_project_scanner.py @@ -15,11 +15,13 @@ sys.path.insert(0, str(ROOT)) from util.project_scanner import ( # noqa: E402 detect_wolf_layout, find_wolf_text_archives, + find_wolf_unpack_gaps, wolf_has_maps, wolf_maps_dir, wolf_maps_packed, wolf_repair_nested_data_dir, wolf_unpack_out_dir, + wolf_unpack_target_dir, ) @@ -114,6 +116,55 @@ class WolfProjectScannerTests(unittest.TestCase): self.assertTrue(info["unpacked"]) self.assertEqual(info["data_dir"], outer) + def test_find_wolf_unpack_gaps_when_mapdata_missing(self): + with tempfile.TemporaryDirectory() as raw: + base = Path(raw) + data = base / "Data" + basic = data / "BasicData" + basic.mkdir(parents=True) + (basic / "CommonEvent.dat").write_bytes(b"") + (data / "BasicData.wolf").write_bytes(b"") + (data / "MapData.wolf").write_bytes(b"") + gaps = find_wolf_unpack_gaps(base) + self.assertEqual(gaps, [data / "MapData.wolf"]) + + def test_find_wolf_unpack_gaps_empty_when_split_archives_unpacked(self): + with tempfile.TemporaryDirectory() as raw: + base = Path(raw) + data = base / "Data" + basic = data / "BasicData" + maps = data / "MapData" + basic.mkdir(parents=True) + maps.mkdir(parents=True) + (basic / "CommonEvent.dat").write_bytes(b"") + (maps / "town.mps").write_bytes(b"") + (data / "BasicData.wolf").write_bytes(b"") + (data / "MapData.wolf").write_bytes(b"") + self.assertEqual(find_wolf_unpack_gaps(base), []) + + def test_detect_wolf_layout_reports_unpack_gaps(self): + with tempfile.TemporaryDirectory() as raw: + base = Path(raw) + data = base / "Data" + basic = data / "BasicData" + basic.mkdir(parents=True) + (basic / "CommonEvent.dat").write_bytes(b"") + map_arc = data / "MapData.wolf" + map_arc.write_bytes(b"") + info = detect_wolf_layout(base) + self.assertFalse(info["unpack_complete"]) + self.assertEqual(info["unpack_gaps"], [map_arc]) + self.assertTrue(info["unpacked"]) + + def test_wolf_unpack_target_dir(self): + with tempfile.TemporaryDirectory() as raw: + base = Path(raw) + data = base / "Data" + data.mkdir() + arc = data / "MapData.wolf" + arc.write_bytes(b"") + self.assertEqual(wolf_unpack_target_dir(base, arc), data / "MapData") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_wolf_unpack_progress.py b/tests/test_wolf_unpack_progress.py index 1eb31f5..c33dd9b 100644 --- a/tests/test_wolf_unpack_progress.py +++ b/tests/test_wolf_unpack_progress.py @@ -7,10 +7,13 @@ from pathlib import Path from unittest.mock import patch from util.wolfdawn import ( + WolfResult, _UNPACK_ARCHIVE_DONE_RE, + _UNPACK_ONE_DONE_RE, _UNPACK_SUMMARY_RE, _run_streaming, count_unpack_archives, + unpack_all, ) @@ -47,6 +50,45 @@ class WolfUnpackProgressTests(unittest.TestCase): (base / "BasicData.wolf").write_bytes(b"") self.assertEqual(count_unpack_archives([base]), 2) + def test_unpack_one_done_regex(self): + line = "unpacked 74 files -> /tmp/out/MapData" + match = _UNPACK_ONE_DONE_RE.match(line) + self.assertIsNotNone(match) + self.assertEqual(match.group("files"), "74") + + def test_unpack_all_uses_separate_subprocess_per_archive(self): + import tempfile + + calls: list[list[str]] = [] + + def fake_run(args, log_fn=None): + calls.append(list(args)) + name = Path(args[1]).name + files = "10" if name.startswith("Basic") else "5" + return WolfResult( + returncode=0, + stdout=f"unpacked {files} files -> /tmp/out\n", + stderr="", + argv=["wolf", *args], + ) + + with tempfile.TemporaryDirectory() as raw: + base = Path(raw) + a = base / "BasicData.wolf" + b = base / "MapData.wolf" + a.write_bytes(b"x") + b.write_bytes(b"x") + out = base / "out" + with patch("util.wolfdawn._run", side_effect=fake_run): + result = unpack_all([a, b], out, progress_total=2) + + self.assertTrue(result.ok) + self.assertEqual(len(calls), 2) + self.assertEqual(calls[0][:2], ["unpack", str(a)]) + self.assertEqual(calls[1][:2], ["unpack", str(b)]) + self.assertTrue(str(out / "BasicData") in calls[0][3]) + self.assertTrue(str(out / "MapData") in calls[1][3]) + def test_run_streaming_emits_unpack_progress(self): events: list[tuple[int, int, str]] = [] diff --git a/util/project_scanner.py b/util/project_scanner.py index 6522d83..b0e1d73 100644 --- a/util/project_scanner.py +++ b/util/project_scanner.py @@ -184,6 +184,46 @@ def _wolf_dir_has_loose_data(data_dir: Path) -> bool: return False +def wolf_unpack_target_dir(game_root: str | Path, archive: str | Path) -> Path: + """Return the folder ``wolf unpack-all`` writes for *archive* (``//``).""" + archive = Path(archive) + return wolf_unpack_out_dir(game_root, archive) / archive.stem + + +def _wolf_archive_unpack_ok(game_root: str | Path, archive: Path) -> bool: + """True when *archive*'s expected unpack folder exists and looks populated.""" + target = wolf_unpack_target_dir(game_root, archive) + if not target.is_dir(): + return False + stem = archive.stem + if stem == "Data": + return _wolf_dir_has_loose_data(target) + if stem == "BasicData": + return (target / "CommonEvent.dat").is_file() or any(target.glob("*.dat")) + if stem == "MapData": + return wolf_has_maps(wolf_unpack_out_dir(game_root, archive)) + try: + return any(target.iterdir()) + except OSError: + return False + + +def find_wolf_unpack_gaps( + game_root: str | Path, + archives: list[Path] | None = None, +) -> list[Path]: + """Return ``.wolf`` archives whose unpack output folder is missing or empty. + + Used to catch interrupted unpacks: a game may already have ``BasicData/`` loose + while ``MapData.wolf`` was never finished, and the old ``unpacked`` flag still + read as true. + """ + root = Path(game_root) + if archives is None: + archives = find_wolf_archives(root) + return [arc for arc in archives if not _wolf_archive_unpack_ok(root, arc)] + + def wolf_unpack_out_dir(game_root: str | Path, archive: str | Path) -> Path: """Return the ``-o`` directory for ``wolf unpack-all`` on *archive*. @@ -245,6 +285,9 @@ def detect_wolf_layout(game_root: str | Path) -> dict: archives : list[Path] of .wolf archives found (may be empty) data_dir : Path to the loose/unpacked Data/ folder, or None unpacked : bool - True when a usable loose Data/ folder exists + unpack_gaps: list[Path] of .wolf archives not yet unpacked (empty when + there are no archives, or every archive has output on disk) + unpack_complete: bool - True when there are no pending unpack gaps basic_data : Path to Data/BasicData (or Data/ when flattened), or None map_data : Path to Data/MapData (or Data/ when flattened), or None """ @@ -255,6 +298,8 @@ def detect_wolf_layout(game_root: str | Path) -> dict: "archives": [], "data_dir": None, "unpacked": False, + "unpack_gaps": [], + "unpack_complete": True, "basic_data": None, "map_data": None, } @@ -269,6 +314,9 @@ def detect_wolf_layout(game_root: str | Path) -> dict: result["engine"] = "WOLF" result["archives"] = archives + gaps = find_wolf_unpack_gaps(root, archives) + result["unpack_gaps"] = gaps + result["unpack_complete"] = not gaps if loose is not None: result["data_dir"] = loose result["unpacked"] = True diff --git a/util/wolfdawn/__init__.py b/util/wolfdawn/__init__.py index 42a04f9..7d6d47f 100644 --- a/util/wolfdawn/__init__.py +++ b/util/wolfdawn/__init__.py @@ -77,6 +77,9 @@ ProgressFn = Callable[[int, int, str], None] _UNPACK_ARCHIVE_DONE_RE = re.compile( r"^\s+(?P\S+\.wolf)\s+->\s+(?P.+?)\s+\((?P\d+) files\)\s*$" ) +_UNPACK_ONE_DONE_RE = re.compile( + r"^unpacked\s+(?P\d+)\s+files\s+->", re.IGNORECASE +) _UNPACK_SUMMARY_RE = re.compile( r"^unpack-all:\s+(?P\d+) archive\(s\),\s+(?P\d+) files\s*$" ) @@ -445,24 +448,39 @@ def _run_streaming( def count_unpack_archives(inputs: Union[PathLike, Iterable[PathLike]]) -> int | None: """Return how many ``.wolf`` archives *inputs* will unpack, or None if unknown.""" + expanded = _expand_wolf_archives(inputs) + if expanded is None: + return None + return len(expanded) + + +def _expand_wolf_archives(inputs: Union[PathLike, Iterable[PathLike]]) -> list[Path] | None: + """Expand directory inputs to ``*.wolf`` files; return None if any path is invalid.""" if isinstance(inputs, (str, Path)): inputs = [inputs] - total = 0 + archives: list[Path] = [] for item in inputs: path = Path(item) if path.is_file(): lower = path.name.lower() if lower.endswith(".wolf") or lower.endswith(".wolf.bak"): - total += 1 + archives.append(path) else: return None elif path.is_dir(): - total += len(list(path.glob("*.wolf"))) + archives.extend(sorted(path.glob("*.wolf"))) else: return None - return total + return archives +def _parse_unpack_one_files(stdout: str) -> str | None: + for line in (stdout or "").splitlines(): + match = _UNPACK_ONE_DONE_RE.match(line.strip()) + if match: + return match.group("files") + return None + # --------------------------------------------------------------------------- # Command wrappers # --------------------------------------------------------------------------- @@ -474,28 +492,90 @@ def unpack_all( progress_fn: ProgressFn | None = None, progress_total: int | None = None, ) -> WolfResult: - """``wolf unpack-all ... -o ``. + """Unpack each ``Name.wolf`` to ``/Name/``. - Each ``Name.wolf`` is unpacked to ``/Name/``. - - When *progress_fn* is set, it receives ``(current, total, label)`` as each - archive finishes. Pass *progress_total* when known; otherwise the archive - count is inferred from ``unpack-all: N archive(s), …`` in the CLI output. + WolfDawn reads each archive entirely into memory (``std::fs::read``), so this + runs ``wolf unpack`` once per archive in a fresh subprocess. That way peak + RAM is freed between large archives instead of stacking in one long + ``unpack-all`` process. """ - if isinstance(inputs, (str, Path)): - inputs = [inputs] - input_list = [_str(p) for p in inputs] - if progress_total is None: - progress_total = count_unpack_archives(inputs) - args = ["unpack-all", *input_list, "-o", _str(out_dir)] - if progress_fn is not None: - return _run_streaming( - args, - log_fn=log_fn, - progress_fn=progress_fn, - progress_total=progress_total, + archives = _expand_wolf_archives(inputs) + if archives is None: + return WolfResult( + returncode=64, + stdout="", + stderr="invalid unpack input path", + argv=["wolf", "unpack", "…"], ) - return _run(args, log_fn=log_fn) + if not archives: + return WolfResult( + returncode=4, + stdout="", + stderr="no .wolf archives found in the given paths", + argv=["wolf", "unpack", "…"], + ) + + total = progress_total if progress_total is not None else len(archives) + out_root = Path(out_dir) + ok = 0 + failed = 0 + total_files = 0 + log_lines: list[str] = [] + last_argv: list[str] = [] + + for index, archive in enumerate(archives): + stem = archive.stem + target = out_root / stem + if progress_fn and total > 0: + _emit_unpack_progress( + progress_fn, + index, + total, + f"Unpacking {archive.name} ({index + 1} of {total}) …", + ) + res = _run(["unpack", _str(archive), "-o", _str(target)], log_fn=log_fn) + last_argv = res.argv + if res.stdout: + log_lines.append(res.stdout.strip()) + if res.ok: + ok += 1 + files = _parse_unpack_one_files(res.stdout) or "?" + try: + total_files += int(files) + except ValueError: + pass + if progress_fn and total > 0: + label = f"Unpacked {archive.name} ({files} files)" + if index + 1 < total: + label += f" — archive {index + 2} of {total} …" + _emit_unpack_progress(progress_fn, index + 1, total, label) + else: + failed += 1 + if log_fn: + log_fn(f" {archive.name} FAILED (exit {res.returncode})") + + summary = f"unpack-all: {ok} archive(s), {total_files} files" + if failed: + summary += f", {failed} failed" + if log_fn: + log_fn(summary) + log_lines.append(summary) + + return WolfResult( + returncode=0 if ok > 0 else 4, + stdout="\n".join(log_lines), + stderr="", + argv=last_argv or ["wolf", "unpack", "…"], + ) + + +def unpack_one( + archive: PathLike, + out_dir: PathLike, + log_fn=None, +) -> WolfResult: + """``wolf unpack -o ``.""" + return _run(["unpack", _str(archive), "-o", _str(out_dir)], log_fn=log_fn) def strings_extract(input_path: PathLike, out_json: PathLike, log_fn=None) -> WolfResult: