Unpack one archive at a time to prevent oom issues

gui/wolf_workflow_tab.py

@@ -92,6 +92,7 @@ from util.paths import PROJECT_ROOT
 from util.project_scanner import (
     detect_wolf_layout,
     find_wolf_text_archives,
+    find_wolf_unpack_gaps,
     list_wolf_json_files,
     wolf_has_maps,
     wolf_maps_dir,
@@ -1077,6 +1078,19 @@ class WolfWorkflowTab(QWidget):
         if self._worker is not None and self._worker.isRunning():
             return
 
+        info = self._layout or detect_wolf_layout(self._game_root)
+        self._layout = info
+        gaps = info.get("unpack_gaps") or []
+        if gaps or (info.get("archives") and not info.get("unpacked")):
+            pending = gaps or info.get("archives") or []
+            self._log(f"Auto-setup: unpacking {len(pending)} archive(s) …")
+            self._unpack(
+                on_done=self._on_auto_unpack_done,
+                interactive=False,
+                archives=pending,
+            )
+            return
+
         manifest = self._read_manifest()
         if manifest and manifest.get("entries"):
             info = self._layout or detect_wolf_layout(self._game_root)
@@ -1102,12 +1116,6 @@ class WolfWorkflowTab(QWidget):
             self._scan_wolf_files()
             return
 
-        info = self._layout or detect_wolf_layout(self._game_root)
-        self._layout = info
-        if info.get("archives") and not info.get("unpacked"):
-            self._log("Auto-setup: unpacking .wolf archives …")
-            self._unpack(on_done=self._on_auto_unpack_done, interactive=False)
-            return
         if info.get("data_dir"):
             self._log("Auto-setup: extracting text into wolf_json/ …")
             self._extract_to_work_dir(on_done=lambda ok, _m: self._scan_wolf_files() if ok else None)
@@ -1118,6 +1126,11 @@ class WolfWorkflowTab(QWidget):
         if not ok:
             return
         self._layout = detect_wolf_layout(self._game_root)
+        gaps = self._layout.get("unpack_gaps") or []
+        if gaps:
+            names = ", ".join(a.name for a in gaps)
+            self._log(f"⚠  Unpack still incomplete: {names}")
+            return
         if self._layout.get("data_dir"):
             self._log("Auto-setup: extracting text into wolf_json/ …")
             self._extract_to_work_dir(on_done=lambda ok2, _m: self._scan_wolf_files() if ok2 else None)
@@ -1401,12 +1414,14 @@ class WolfWorkflowTab(QWidget):
 
         self._run_task(task)
 
-    def _unpack(self, on_done=None, *, interactive: bool = True):
+    def _unpack(self, on_done=None, *, interactive: bool = True, archives=None):
         if not self._require_root():
             return
         root = Path(self._game_root)
         info = detect_wolf_layout(root)
-        archives = info["archives"]
+        if archives is None:
+            gaps = info.get("unpack_gaps") or find_wolf_unpack_gaps(root)
+            archives = gaps if gaps else info["archives"]
         if not archives:
             self._log("Unpack: no .wolf archives found (already unpacked?).")
             if interactive:

tests/test_wolf_project_scanner.py

@@ -15,11 +15,13 @@ sys.path.insert(0, str(ROOT))
 from util.project_scanner import (  # noqa: E402
     detect_wolf_layout,
     find_wolf_text_archives,
+    find_wolf_unpack_gaps,
     wolf_has_maps,
     wolf_maps_dir,
     wolf_maps_packed,
     wolf_repair_nested_data_dir,
     wolf_unpack_out_dir,
+    wolf_unpack_target_dir,
 )
 
 
@@ -114,6 +116,55 @@ class WolfProjectScannerTests(unittest.TestCase):
             self.assertTrue(info["unpacked"])
             self.assertEqual(info["data_dir"], outer)
 
+    def test_find_wolf_unpack_gaps_when_mapdata_missing(self):
+        with tempfile.TemporaryDirectory() as raw:
+            base = Path(raw)
+            data = base / "Data"
+            basic = data / "BasicData"
+            basic.mkdir(parents=True)
+            (basic / "CommonEvent.dat").write_bytes(b"")
+            (data / "BasicData.wolf").write_bytes(b"")
+            (data / "MapData.wolf").write_bytes(b"")
+            gaps = find_wolf_unpack_gaps(base)
+            self.assertEqual(gaps, [data / "MapData.wolf"])
+
+    def test_find_wolf_unpack_gaps_empty_when_split_archives_unpacked(self):
+        with tempfile.TemporaryDirectory() as raw:
+            base = Path(raw)
+            data = base / "Data"
+            basic = data / "BasicData"
+            maps = data / "MapData"
+            basic.mkdir(parents=True)
+            maps.mkdir(parents=True)
+            (basic / "CommonEvent.dat").write_bytes(b"")
+            (maps / "town.mps").write_bytes(b"")
+            (data / "BasicData.wolf").write_bytes(b"")
+            (data / "MapData.wolf").write_bytes(b"")
+            self.assertEqual(find_wolf_unpack_gaps(base), [])
+
+    def test_detect_wolf_layout_reports_unpack_gaps(self):
+        with tempfile.TemporaryDirectory() as raw:
+            base = Path(raw)
+            data = base / "Data"
+            basic = data / "BasicData"
+            basic.mkdir(parents=True)
+            (basic / "CommonEvent.dat").write_bytes(b"")
+            map_arc = data / "MapData.wolf"
+            map_arc.write_bytes(b"")
+            info = detect_wolf_layout(base)
+            self.assertFalse(info["unpack_complete"])
+            self.assertEqual(info["unpack_gaps"], [map_arc])
+            self.assertTrue(info["unpacked"])
+
+    def test_wolf_unpack_target_dir(self):
+        with tempfile.TemporaryDirectory() as raw:
+            base = Path(raw)
+            data = base / "Data"
+            data.mkdir()
+            arc = data / "MapData.wolf"
+            arc.write_bytes(b"")
+            self.assertEqual(wolf_unpack_target_dir(base, arc), data / "MapData")
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_wolf_unpack_progress.py

@@ -7,10 +7,13 @@ from pathlib import Path
 from unittest.mock import patch
 
 from util.wolfdawn import (
+    WolfResult,
     _UNPACK_ARCHIVE_DONE_RE,
+    _UNPACK_ONE_DONE_RE,
     _UNPACK_SUMMARY_RE,
     _run_streaming,
     count_unpack_archives,
+    unpack_all,
 )
 
 
@@ -47,6 +50,45 @@ class WolfUnpackProgressTests(unittest.TestCase):
             (base / "BasicData.wolf").write_bytes(b"")
             self.assertEqual(count_unpack_archives([base]), 2)
 
+    def test_unpack_one_done_regex(self):
+        line = "unpacked 74 files -> /tmp/out/MapData"
+        match = _UNPACK_ONE_DONE_RE.match(line)
+        self.assertIsNotNone(match)
+        self.assertEqual(match.group("files"), "74")
+
+    def test_unpack_all_uses_separate_subprocess_per_archive(self):
+        import tempfile
+
+        calls: list[list[str]] = []
+
+        def fake_run(args, log_fn=None):
+            calls.append(list(args))
+            name = Path(args[1]).name
+            files = "10" if name.startswith("Basic") else "5"
+            return WolfResult(
+                returncode=0,
+                stdout=f"unpacked {files} files -> /tmp/out\n",
+                stderr="",
+                argv=["wolf", *args],
+            )
+
+        with tempfile.TemporaryDirectory() as raw:
+            base = Path(raw)
+            a = base / "BasicData.wolf"
+            b = base / "MapData.wolf"
+            a.write_bytes(b"x")
+            b.write_bytes(b"x")
+            out = base / "out"
+            with patch("util.wolfdawn._run", side_effect=fake_run):
+                result = unpack_all([a, b], out, progress_total=2)
+
+        self.assertTrue(result.ok)
+        self.assertEqual(len(calls), 2)
+        self.assertEqual(calls[0][:2], ["unpack", str(a)])
+        self.assertEqual(calls[1][:2], ["unpack", str(b)])
+        self.assertTrue(str(out / "BasicData") in calls[0][3])
+        self.assertTrue(str(out / "MapData") in calls[1][3])
+
     def test_run_streaming_emits_unpack_progress(self):
         events: list[tuple[int, int, str]] = []
 

util/project_scanner.py

@@ -184,6 +184,46 @@ def _wolf_dir_has_loose_data(data_dir: Path) -> bool:
     return False
 
 
+def wolf_unpack_target_dir(game_root: str | Path, archive: str | Path) -> Path:
+    """Return the folder ``wolf unpack-all`` writes for *archive* (``<out>/<stem>/``)."""
+    archive = Path(archive)
+    return wolf_unpack_out_dir(game_root, archive) / archive.stem
+
+
+def _wolf_archive_unpack_ok(game_root: str | Path, archive: Path) -> bool:
+    """True when *archive*'s expected unpack folder exists and looks populated."""
+    target = wolf_unpack_target_dir(game_root, archive)
+    if not target.is_dir():
+        return False
+    stem = archive.stem
+    if stem == "Data":
+        return _wolf_dir_has_loose_data(target)
+    if stem == "BasicData":
+        return (target / "CommonEvent.dat").is_file() or any(target.glob("*.dat"))
+    if stem == "MapData":
+        return wolf_has_maps(wolf_unpack_out_dir(game_root, archive))
+    try:
+        return any(target.iterdir())
+    except OSError:
+        return False
+
+
+def find_wolf_unpack_gaps(
+    game_root: str | Path,
+    archives: list[Path] | None = None,
+) -> list[Path]:
+    """Return ``.wolf`` archives whose unpack output folder is missing or empty.
+
+    Used to catch interrupted unpacks: a game may already have ``BasicData/`` loose
+    while ``MapData.wolf`` was never finished, and the old ``unpacked`` flag still
+    read as true.
+    """
+    root = Path(game_root)
+    if archives is None:
+        archives = find_wolf_archives(root)
+    return [arc for arc in archives if not _wolf_archive_unpack_ok(root, arc)]
+
+
 def wolf_unpack_out_dir(game_root: str | Path, archive: str | Path) -> Path:
     """Return the ``-o`` directory for ``wolf unpack-all`` on *archive*.
 
@@ -245,6 +285,9 @@ def detect_wolf_layout(game_root: str | Path) -> dict:
         archives   : list[Path] of .wolf archives found (may be empty)
         data_dir   : Path to the loose/unpacked Data/ folder, or None
         unpacked   : bool - True when a usable loose Data/ folder exists
+        unpack_gaps: list[Path] of .wolf archives not yet unpacked (empty when
+                     there are no archives, or every archive has output on disk)
+        unpack_complete: bool - True when there are no pending unpack gaps
         basic_data : Path to Data/BasicData (or Data/ when flattened), or None
         map_data   : Path to Data/MapData (or Data/ when flattened), or None
     """
@@ -255,6 +298,8 @@ def detect_wolf_layout(game_root: str | Path) -> dict:
         "archives": [],
         "data_dir": None,
         "unpacked": False,
+        "unpack_gaps": [],
+        "unpack_complete": True,
         "basic_data": None,
         "map_data": None,
     }
@@ -269,6 +314,9 @@ def detect_wolf_layout(game_root: str | Path) -> dict:
 
     result["engine"] = "WOLF"
     result["archives"] = archives
+    gaps = find_wolf_unpack_gaps(root, archives)
+    result["unpack_gaps"] = gaps
+    result["unpack_complete"] = not gaps
     if loose is not None:
         result["data_dir"] = loose
         result["unpacked"] = True

util/wolfdawn/__init__.py

@@ -77,6 +77,9 @@ ProgressFn = Callable[[int, int, str], None]
 _UNPACK_ARCHIVE_DONE_RE = re.compile(
     r"^\s+(?P<name>\S+\.wolf)\s+->\s+(?P<dest>.+?)\s+\((?P<files>\d+) files\)\s*$"
 )
+_UNPACK_ONE_DONE_RE = re.compile(
+    r"^unpacked\s+(?P<files>\d+)\s+files\s+->", re.IGNORECASE
+)
 _UNPACK_SUMMARY_RE = re.compile(
     r"^unpack-all:\s+(?P<archives>\d+) archive\(s\),\s+(?P<files>\d+) files\s*$"
 )
@@ -445,24 +448,39 @@ def _run_streaming(
 
 def count_unpack_archives(inputs: Union[PathLike, Iterable[PathLike]]) -> int | None:
     """Return how many ``.wolf`` archives *inputs* will unpack, or None if unknown."""
+    expanded = _expand_wolf_archives(inputs)
+    if expanded is None:
+        return None
+    return len(expanded)
+
+
+def _expand_wolf_archives(inputs: Union[PathLike, Iterable[PathLike]]) -> list[Path] | None:
+    """Expand directory inputs to ``*.wolf`` files; return None if any path is invalid."""
     if isinstance(inputs, (str, Path)):
         inputs = [inputs]
-    total = 0
+    archives: list[Path] = []
     for item in inputs:
         path = Path(item)
         if path.is_file():
             lower = path.name.lower()
             if lower.endswith(".wolf") or lower.endswith(".wolf.bak"):
-                total += 1
+                archives.append(path)
             else:
                 return None
         elif path.is_dir():
-            total += len(list(path.glob("*.wolf")))
+            archives.extend(sorted(path.glob("*.wolf")))
         else:
             return None
-    return total
+    return archives
 
 
+def _parse_unpack_one_files(stdout: str) -> str | None:
+    for line in (stdout or "").splitlines():
+        match = _UNPACK_ONE_DONE_RE.match(line.strip())
+        if match:
+            return match.group("files")
+    return None
+
 # ---------------------------------------------------------------------------
 # Command wrappers
 # ---------------------------------------------------------------------------
@@ -474,28 +492,90 @@ def unpack_all(
     progress_fn: ProgressFn | None = None,
     progress_total: int | None = None,
 ) -> WolfResult:
-    """``wolf unpack-all <data-dir|archive.wolf>... -o <out-dir>``.
+    """Unpack each ``Name.wolf`` to ``<out-dir>/Name/``.
 
-    Each ``Name.wolf`` is unpacked to ``<out-dir>/Name/``.
+    WolfDawn reads each archive entirely into memory (``std::fs::read``), so this
-
+    runs ``wolf unpack`` once per archive in a fresh subprocess. That way peak
-    When *progress_fn* is set, it receives ``(current, total, label)`` as each
+    RAM is freed between large archives instead of stacking in one long
-    archive finishes. Pass *progress_total* when known; otherwise the archive
+    ``unpack-all`` process.
-    count is inferred from ``unpack-all: N archive(s), …`` in the CLI output.
     """
-    if isinstance(inputs, (str, Path)):
+    archives = _expand_wolf_archives(inputs)
-        inputs = [inputs]
+    if archives is None:
-    input_list = [_str(p) for p in inputs]
+        return WolfResult(
-    if progress_total is None:
+            returncode=64,
-        progress_total = count_unpack_archives(inputs)
+            stdout="",
-    args = ["unpack-all", *input_list, "-o", _str(out_dir)]
+            stderr="invalid unpack input path",
-    if progress_fn is not None:
+            argv=["wolf", "unpack", "…"],
-        return _run_streaming(
-            args,
-            log_fn=log_fn,
-            progress_fn=progress_fn,
-            progress_total=progress_total,
         )
-    return _run(args, log_fn=log_fn)
+    if not archives:
+        return WolfResult(
+            returncode=4,
+            stdout="",
+            stderr="no .wolf archives found in the given paths",
+            argv=["wolf", "unpack", "…"],
+        )
+
+    total = progress_total if progress_total is not None else len(archives)
+    out_root = Path(out_dir)
+    ok = 0
+    failed = 0
+    total_files = 0
+    log_lines: list[str] = []
+    last_argv: list[str] = []
+
+    for index, archive in enumerate(archives):
+        stem = archive.stem
+        target = out_root / stem
+        if progress_fn and total > 0:
+            _emit_unpack_progress(
+                progress_fn,
+                index,
+                total,
+                f"Unpacking {archive.name} ({index + 1} of {total}) …",
+            )
+        res = _run(["unpack", _str(archive), "-o", _str(target)], log_fn=log_fn)
+        last_argv = res.argv
+        if res.stdout:
+            log_lines.append(res.stdout.strip())
+        if res.ok:
+            ok += 1
+            files = _parse_unpack_one_files(res.stdout) or "?"
+            try:
+                total_files += int(files)
+            except ValueError:
+                pass
+            if progress_fn and total > 0:
+                label = f"Unpacked {archive.name} ({files} files)"
+                if index + 1 < total:
+                    label += f" — archive {index + 2} of {total} …"
+                _emit_unpack_progress(progress_fn, index + 1, total, label)
+        else:
+            failed += 1
+            if log_fn:
+                log_fn(f"  {archive.name} FAILED (exit {res.returncode})")
+
+    summary = f"unpack-all: {ok} archive(s), {total_files} files"
+    if failed:
+        summary += f", {failed} failed"
+    if log_fn:
+        log_fn(summary)
+    log_lines.append(summary)
+
+    return WolfResult(
+        returncode=0 if ok > 0 else 4,
+        stdout="\n".join(log_lines),
+        stderr="",
+        argv=last_argv or ["wolf", "unpack", "…"],
+    )
+
+
+def unpack_one(
+    archive: PathLike,
+    out_dir: PathLike,
+    log_fn=None,
+) -> WolfResult:
+    """``wolf unpack <archive.wolf> -o <out-dir>``."""
+    return _run(["unpack", _str(archive), "-o", _str(out_dir)], log_fn=log_fn)
 
 
 def strings_extract(input_path: PathLike, out_json: PathLike, log_fn=None) -> WolfResult: