Unpack one archive at a time to prevent oom issues
This commit is contained in:
parent
7a1109d8be
commit
1bb89a8b48
5 changed files with 267 additions and 31 deletions
|
|
@ -92,6 +92,7 @@ from util.paths import PROJECT_ROOT
|
|||
from util.project_scanner import (
|
||||
detect_wolf_layout,
|
||||
find_wolf_text_archives,
|
||||
find_wolf_unpack_gaps,
|
||||
list_wolf_json_files,
|
||||
wolf_has_maps,
|
||||
wolf_maps_dir,
|
||||
|
|
@ -1077,6 +1078,19 @@ class WolfWorkflowTab(QWidget):
|
|||
if self._worker is not None and self._worker.isRunning():
|
||||
return
|
||||
|
||||
info = self._layout or detect_wolf_layout(self._game_root)
|
||||
self._layout = info
|
||||
gaps = info.get("unpack_gaps") or []
|
||||
if gaps or (info.get("archives") and not info.get("unpacked")):
|
||||
pending = gaps or info.get("archives") or []
|
||||
self._log(f"Auto-setup: unpacking {len(pending)} archive(s) …")
|
||||
self._unpack(
|
||||
on_done=self._on_auto_unpack_done,
|
||||
interactive=False,
|
||||
archives=pending,
|
||||
)
|
||||
return
|
||||
|
||||
manifest = self._read_manifest()
|
||||
if manifest and manifest.get("entries"):
|
||||
info = self._layout or detect_wolf_layout(self._game_root)
|
||||
|
|
@ -1102,12 +1116,6 @@ class WolfWorkflowTab(QWidget):
|
|||
self._scan_wolf_files()
|
||||
return
|
||||
|
||||
info = self._layout or detect_wolf_layout(self._game_root)
|
||||
self._layout = info
|
||||
if info.get("archives") and not info.get("unpacked"):
|
||||
self._log("Auto-setup: unpacking .wolf archives …")
|
||||
self._unpack(on_done=self._on_auto_unpack_done, interactive=False)
|
||||
return
|
||||
if info.get("data_dir"):
|
||||
self._log("Auto-setup: extracting text into wolf_json/ …")
|
||||
self._extract_to_work_dir(on_done=lambda ok, _m: self._scan_wolf_files() if ok else None)
|
||||
|
|
@ -1118,6 +1126,11 @@ class WolfWorkflowTab(QWidget):
|
|||
if not ok:
|
||||
return
|
||||
self._layout = detect_wolf_layout(self._game_root)
|
||||
gaps = self._layout.get("unpack_gaps") or []
|
||||
if gaps:
|
||||
names = ", ".join(a.name for a in gaps)
|
||||
self._log(f"⚠ Unpack still incomplete: {names}")
|
||||
return
|
||||
if self._layout.get("data_dir"):
|
||||
self._log("Auto-setup: extracting text into wolf_json/ …")
|
||||
self._extract_to_work_dir(on_done=lambda ok2, _m: self._scan_wolf_files() if ok2 else None)
|
||||
|
|
@ -1401,12 +1414,14 @@ class WolfWorkflowTab(QWidget):
|
|||
|
||||
self._run_task(task)
|
||||
|
||||
def _unpack(self, on_done=None, *, interactive: bool = True):
|
||||
def _unpack(self, on_done=None, *, interactive: bool = True, archives=None):
|
||||
if not self._require_root():
|
||||
return
|
||||
root = Path(self._game_root)
|
||||
info = detect_wolf_layout(root)
|
||||
archives = info["archives"]
|
||||
if archives is None:
|
||||
gaps = info.get("unpack_gaps") or find_wolf_unpack_gaps(root)
|
||||
archives = gaps if gaps else info["archives"]
|
||||
if not archives:
|
||||
self._log("Unpack: no .wolf archives found (already unpacked?).")
|
||||
if interactive:
|
||||
|
|
|
|||
|
|
@ -15,11 +15,13 @@ sys.path.insert(0, str(ROOT))
|
|||
from util.project_scanner import ( # noqa: E402
|
||||
detect_wolf_layout,
|
||||
find_wolf_text_archives,
|
||||
find_wolf_unpack_gaps,
|
||||
wolf_has_maps,
|
||||
wolf_maps_dir,
|
||||
wolf_maps_packed,
|
||||
wolf_repair_nested_data_dir,
|
||||
wolf_unpack_out_dir,
|
||||
wolf_unpack_target_dir,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -114,6 +116,55 @@ class WolfProjectScannerTests(unittest.TestCase):
|
|||
self.assertTrue(info["unpacked"])
|
||||
self.assertEqual(info["data_dir"], outer)
|
||||
|
||||
def test_find_wolf_unpack_gaps_when_mapdata_missing(self):
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
base = Path(raw)
|
||||
data = base / "Data"
|
||||
basic = data / "BasicData"
|
||||
basic.mkdir(parents=True)
|
||||
(basic / "CommonEvent.dat").write_bytes(b"")
|
||||
(data / "BasicData.wolf").write_bytes(b"")
|
||||
(data / "MapData.wolf").write_bytes(b"")
|
||||
gaps = find_wolf_unpack_gaps(base)
|
||||
self.assertEqual(gaps, [data / "MapData.wolf"])
|
||||
|
||||
def test_find_wolf_unpack_gaps_empty_when_split_archives_unpacked(self):
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
base = Path(raw)
|
||||
data = base / "Data"
|
||||
basic = data / "BasicData"
|
||||
maps = data / "MapData"
|
||||
basic.mkdir(parents=True)
|
||||
maps.mkdir(parents=True)
|
||||
(basic / "CommonEvent.dat").write_bytes(b"")
|
||||
(maps / "town.mps").write_bytes(b"")
|
||||
(data / "BasicData.wolf").write_bytes(b"")
|
||||
(data / "MapData.wolf").write_bytes(b"")
|
||||
self.assertEqual(find_wolf_unpack_gaps(base), [])
|
||||
|
||||
def test_detect_wolf_layout_reports_unpack_gaps(self):
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
base = Path(raw)
|
||||
data = base / "Data"
|
||||
basic = data / "BasicData"
|
||||
basic.mkdir(parents=True)
|
||||
(basic / "CommonEvent.dat").write_bytes(b"")
|
||||
map_arc = data / "MapData.wolf"
|
||||
map_arc.write_bytes(b"")
|
||||
info = detect_wolf_layout(base)
|
||||
self.assertFalse(info["unpack_complete"])
|
||||
self.assertEqual(info["unpack_gaps"], [map_arc])
|
||||
self.assertTrue(info["unpacked"])
|
||||
|
||||
def test_wolf_unpack_target_dir(self):
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
base = Path(raw)
|
||||
data = base / "Data"
|
||||
data.mkdir()
|
||||
arc = data / "MapData.wolf"
|
||||
arc.write_bytes(b"")
|
||||
self.assertEqual(wolf_unpack_target_dir(base, arc), data / "MapData")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -7,10 +7,13 @@ from pathlib import Path
|
|||
from unittest.mock import patch
|
||||
|
||||
from util.wolfdawn import (
|
||||
WolfResult,
|
||||
_UNPACK_ARCHIVE_DONE_RE,
|
||||
_UNPACK_ONE_DONE_RE,
|
||||
_UNPACK_SUMMARY_RE,
|
||||
_run_streaming,
|
||||
count_unpack_archives,
|
||||
unpack_all,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -47,6 +50,45 @@ class WolfUnpackProgressTests(unittest.TestCase):
|
|||
(base / "BasicData.wolf").write_bytes(b"")
|
||||
self.assertEqual(count_unpack_archives([base]), 2)
|
||||
|
||||
def test_unpack_one_done_regex(self):
|
||||
line = "unpacked 74 files -> /tmp/out/MapData"
|
||||
match = _UNPACK_ONE_DONE_RE.match(line)
|
||||
self.assertIsNotNone(match)
|
||||
self.assertEqual(match.group("files"), "74")
|
||||
|
||||
def test_unpack_all_uses_separate_subprocess_per_archive(self):
|
||||
import tempfile
|
||||
|
||||
calls: list[list[str]] = []
|
||||
|
||||
def fake_run(args, log_fn=None):
|
||||
calls.append(list(args))
|
||||
name = Path(args[1]).name
|
||||
files = "10" if name.startswith("Basic") else "5"
|
||||
return WolfResult(
|
||||
returncode=0,
|
||||
stdout=f"unpacked {files} files -> /tmp/out\n",
|
||||
stderr="",
|
||||
argv=["wolf", *args],
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
base = Path(raw)
|
||||
a = base / "BasicData.wolf"
|
||||
b = base / "MapData.wolf"
|
||||
a.write_bytes(b"x")
|
||||
b.write_bytes(b"x")
|
||||
out = base / "out"
|
||||
with patch("util.wolfdawn._run", side_effect=fake_run):
|
||||
result = unpack_all([a, b], out, progress_total=2)
|
||||
|
||||
self.assertTrue(result.ok)
|
||||
self.assertEqual(len(calls), 2)
|
||||
self.assertEqual(calls[0][:2], ["unpack", str(a)])
|
||||
self.assertEqual(calls[1][:2], ["unpack", str(b)])
|
||||
self.assertTrue(str(out / "BasicData") in calls[0][3])
|
||||
self.assertTrue(str(out / "MapData") in calls[1][3])
|
||||
|
||||
def test_run_streaming_emits_unpack_progress(self):
|
||||
events: list[tuple[int, int, str]] = []
|
||||
|
||||
|
|
|
|||
|
|
@ -184,6 +184,46 @@ def _wolf_dir_has_loose_data(data_dir: Path) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def wolf_unpack_target_dir(game_root: str | Path, archive: str | Path) -> Path:
|
||||
"""Return the folder ``wolf unpack-all`` writes for *archive* (``<out>/<stem>/``)."""
|
||||
archive = Path(archive)
|
||||
return wolf_unpack_out_dir(game_root, archive) / archive.stem
|
||||
|
||||
|
||||
def _wolf_archive_unpack_ok(game_root: str | Path, archive: Path) -> bool:
|
||||
"""True when *archive*'s expected unpack folder exists and looks populated."""
|
||||
target = wolf_unpack_target_dir(game_root, archive)
|
||||
if not target.is_dir():
|
||||
return False
|
||||
stem = archive.stem
|
||||
if stem == "Data":
|
||||
return _wolf_dir_has_loose_data(target)
|
||||
if stem == "BasicData":
|
||||
return (target / "CommonEvent.dat").is_file() or any(target.glob("*.dat"))
|
||||
if stem == "MapData":
|
||||
return wolf_has_maps(wolf_unpack_out_dir(game_root, archive))
|
||||
try:
|
||||
return any(target.iterdir())
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def find_wolf_unpack_gaps(
|
||||
game_root: str | Path,
|
||||
archives: list[Path] | None = None,
|
||||
) -> list[Path]:
|
||||
"""Return ``.wolf`` archives whose unpack output folder is missing or empty.
|
||||
|
||||
Used to catch interrupted unpacks: a game may already have ``BasicData/`` loose
|
||||
while ``MapData.wolf`` was never finished, and the old ``unpacked`` flag still
|
||||
read as true.
|
||||
"""
|
||||
root = Path(game_root)
|
||||
if archives is None:
|
||||
archives = find_wolf_archives(root)
|
||||
return [arc for arc in archives if not _wolf_archive_unpack_ok(root, arc)]
|
||||
|
||||
|
||||
def wolf_unpack_out_dir(game_root: str | Path, archive: str | Path) -> Path:
|
||||
"""Return the ``-o`` directory for ``wolf unpack-all`` on *archive*.
|
||||
|
||||
|
|
@ -245,6 +285,9 @@ def detect_wolf_layout(game_root: str | Path) -> dict:
|
|||
archives : list[Path] of .wolf archives found (may be empty)
|
||||
data_dir : Path to the loose/unpacked Data/ folder, or None
|
||||
unpacked : bool - True when a usable loose Data/ folder exists
|
||||
unpack_gaps: list[Path] of .wolf archives not yet unpacked (empty when
|
||||
there are no archives, or every archive has output on disk)
|
||||
unpack_complete: bool - True when there are no pending unpack gaps
|
||||
basic_data : Path to Data/BasicData (or Data/ when flattened), or None
|
||||
map_data : Path to Data/MapData (or Data/ when flattened), or None
|
||||
"""
|
||||
|
|
@ -255,6 +298,8 @@ def detect_wolf_layout(game_root: str | Path) -> dict:
|
|||
"archives": [],
|
||||
"data_dir": None,
|
||||
"unpacked": False,
|
||||
"unpack_gaps": [],
|
||||
"unpack_complete": True,
|
||||
"basic_data": None,
|
||||
"map_data": None,
|
||||
}
|
||||
|
|
@ -269,6 +314,9 @@ def detect_wolf_layout(game_root: str | Path) -> dict:
|
|||
|
||||
result["engine"] = "WOLF"
|
||||
result["archives"] = archives
|
||||
gaps = find_wolf_unpack_gaps(root, archives)
|
||||
result["unpack_gaps"] = gaps
|
||||
result["unpack_complete"] = not gaps
|
||||
if loose is not None:
|
||||
result["data_dir"] = loose
|
||||
result["unpacked"] = True
|
||||
|
|
|
|||
|
|
@ -77,6 +77,9 @@ ProgressFn = Callable[[int, int, str], None]
|
|||
_UNPACK_ARCHIVE_DONE_RE = re.compile(
|
||||
r"^\s+(?P<name>\S+\.wolf)\s+->\s+(?P<dest>.+?)\s+\((?P<files>\d+) files\)\s*$"
|
||||
)
|
||||
_UNPACK_ONE_DONE_RE = re.compile(
|
||||
r"^unpacked\s+(?P<files>\d+)\s+files\s+->", re.IGNORECASE
|
||||
)
|
||||
_UNPACK_SUMMARY_RE = re.compile(
|
||||
r"^unpack-all:\s+(?P<archives>\d+) archive\(s\),\s+(?P<files>\d+) files\s*$"
|
||||
)
|
||||
|
|
@ -445,24 +448,39 @@ def _run_streaming(
|
|||
|
||||
def count_unpack_archives(inputs: Union[PathLike, Iterable[PathLike]]) -> int | None:
|
||||
"""Return how many ``.wolf`` archives *inputs* will unpack, or None if unknown."""
|
||||
expanded = _expand_wolf_archives(inputs)
|
||||
if expanded is None:
|
||||
return None
|
||||
return len(expanded)
|
||||
|
||||
|
||||
def _expand_wolf_archives(inputs: Union[PathLike, Iterable[PathLike]]) -> list[Path] | None:
|
||||
"""Expand directory inputs to ``*.wolf`` files; return None if any path is invalid."""
|
||||
if isinstance(inputs, (str, Path)):
|
||||
inputs = [inputs]
|
||||
total = 0
|
||||
archives: list[Path] = []
|
||||
for item in inputs:
|
||||
path = Path(item)
|
||||
if path.is_file():
|
||||
lower = path.name.lower()
|
||||
if lower.endswith(".wolf") or lower.endswith(".wolf.bak"):
|
||||
total += 1
|
||||
archives.append(path)
|
||||
else:
|
||||
return None
|
||||
elif path.is_dir():
|
||||
total += len(list(path.glob("*.wolf")))
|
||||
archives.extend(sorted(path.glob("*.wolf")))
|
||||
else:
|
||||
return None
|
||||
return total
|
||||
return archives
|
||||
|
||||
|
||||
def _parse_unpack_one_files(stdout: str) -> str | None:
|
||||
for line in (stdout or "").splitlines():
|
||||
match = _UNPACK_ONE_DONE_RE.match(line.strip())
|
||||
if match:
|
||||
return match.group("files")
|
||||
return None
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Command wrappers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -474,28 +492,90 @@ def unpack_all(
|
|||
progress_fn: ProgressFn | None = None,
|
||||
progress_total: int | None = None,
|
||||
) -> WolfResult:
|
||||
"""``wolf unpack-all <data-dir|archive.wolf>... -o <out-dir>``.
|
||||
"""Unpack each ``Name.wolf`` to ``<out-dir>/Name/``.
|
||||
|
||||
Each ``Name.wolf`` is unpacked to ``<out-dir>/Name/``.
|
||||
|
||||
When *progress_fn* is set, it receives ``(current, total, label)`` as each
|
||||
archive finishes. Pass *progress_total* when known; otherwise the archive
|
||||
count is inferred from ``unpack-all: N archive(s), …`` in the CLI output.
|
||||
WolfDawn reads each archive entirely into memory (``std::fs::read``), so this
|
||||
runs ``wolf unpack`` once per archive in a fresh subprocess. That way peak
|
||||
RAM is freed between large archives instead of stacking in one long
|
||||
``unpack-all`` process.
|
||||
"""
|
||||
if isinstance(inputs, (str, Path)):
|
||||
inputs = [inputs]
|
||||
input_list = [_str(p) for p in inputs]
|
||||
if progress_total is None:
|
||||
progress_total = count_unpack_archives(inputs)
|
||||
args = ["unpack-all", *input_list, "-o", _str(out_dir)]
|
||||
if progress_fn is not None:
|
||||
return _run_streaming(
|
||||
args,
|
||||
log_fn=log_fn,
|
||||
progress_fn=progress_fn,
|
||||
progress_total=progress_total,
|
||||
archives = _expand_wolf_archives(inputs)
|
||||
if archives is None:
|
||||
return WolfResult(
|
||||
returncode=64,
|
||||
stdout="",
|
||||
stderr="invalid unpack input path",
|
||||
argv=["wolf", "unpack", "…"],
|
||||
)
|
||||
return _run(args, log_fn=log_fn)
|
||||
if not archives:
|
||||
return WolfResult(
|
||||
returncode=4,
|
||||
stdout="",
|
||||
stderr="no .wolf archives found in the given paths",
|
||||
argv=["wolf", "unpack", "…"],
|
||||
)
|
||||
|
||||
total = progress_total if progress_total is not None else len(archives)
|
||||
out_root = Path(out_dir)
|
||||
ok = 0
|
||||
failed = 0
|
||||
total_files = 0
|
||||
log_lines: list[str] = []
|
||||
last_argv: list[str] = []
|
||||
|
||||
for index, archive in enumerate(archives):
|
||||
stem = archive.stem
|
||||
target = out_root / stem
|
||||
if progress_fn and total > 0:
|
||||
_emit_unpack_progress(
|
||||
progress_fn,
|
||||
index,
|
||||
total,
|
||||
f"Unpacking {archive.name} ({index + 1} of {total}) …",
|
||||
)
|
||||
res = _run(["unpack", _str(archive), "-o", _str(target)], log_fn=log_fn)
|
||||
last_argv = res.argv
|
||||
if res.stdout:
|
||||
log_lines.append(res.stdout.strip())
|
||||
if res.ok:
|
||||
ok += 1
|
||||
files = _parse_unpack_one_files(res.stdout) or "?"
|
||||
try:
|
||||
total_files += int(files)
|
||||
except ValueError:
|
||||
pass
|
||||
if progress_fn and total > 0:
|
||||
label = f"Unpacked {archive.name} ({files} files)"
|
||||
if index + 1 < total:
|
||||
label += f" — archive {index + 2} of {total} …"
|
||||
_emit_unpack_progress(progress_fn, index + 1, total, label)
|
||||
else:
|
||||
failed += 1
|
||||
if log_fn:
|
||||
log_fn(f" {archive.name} FAILED (exit {res.returncode})")
|
||||
|
||||
summary = f"unpack-all: {ok} archive(s), {total_files} files"
|
||||
if failed:
|
||||
summary += f", {failed} failed"
|
||||
if log_fn:
|
||||
log_fn(summary)
|
||||
log_lines.append(summary)
|
||||
|
||||
return WolfResult(
|
||||
returncode=0 if ok > 0 else 4,
|
||||
stdout="\n".join(log_lines),
|
||||
stderr="",
|
||||
argv=last_argv or ["wolf", "unpack", "…"],
|
||||
)
|
||||
|
||||
|
||||
def unpack_one(
|
||||
archive: PathLike,
|
||||
out_dir: PathLike,
|
||||
log_fn=None,
|
||||
) -> WolfResult:
|
||||
"""``wolf unpack <archive.wolf> -o <out-dir>``."""
|
||||
return _run(["unpack", _str(archive), "-o", _str(out_dir)], log_fn=log_fn)
|
||||
|
||||
|
||||
def strings_extract(input_path: PathLike, out_json: PathLike, log_fn=None) -> WolfResult:
|
||||
|
|
|
|||
Loading…
Reference in a new issue