dazed-mods/worldbox-observer-mod
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
worldbox-observer-mod/scripts/harness-run.sh

295 lines
8.6 KiB
Bash
Executable file
Raw Blame History

#!/usr/bin/env bash
# Run an IdleSpectator agent harness scenario (or raw JSONL commands).
# Starts WorldBox via Steam when it is not already running.
#
# Usage:
# ./scripts/harness-run.sh # default: critical_smoke
# ./scripts/harness-run.sh critical_smoke
# ./scripts/harness-run.sh regression # full suite (named scenarios)
# ./scripts/harness-run.sh --repeat 3 # run scenario 3 times (all must PASS)
# ./scripts/harness-run.sh tip_match
# ./scripts/harness-run.sh smoke
# ./scripts/harness-run.sh --cmd '{"id":"1","action":"snapshot"}'
# ./scripts/harness-run.sh --no-launch smoke
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
MOD="${IDLE_SPECTATOR_MOD:-$ROOT/IdleSpectator}"
HARNESS="$MOD/.harness"
LOG="${WORLDBOX_LOG:-$HOME/.config/unity3d/mkarpenko/WorldBox/Player.log}"
TIMEOUT_SEC="${HARNESS_TIMEOUT:-90}"
STEAM_APP_ID="${WORLDBOX_APP_ID:-1206560}"
BOOT_TIMEOUT_SEC="${HARNESS_BOOT_TIMEOUT:-180}"
DO_LAUNCH=1
REPEAT=1
# Full regression gate (battle / map-gen intentionally excluded).
REGRESSION_SCENARIOS=(
critical_smoke
settings_full
ghost_guard
retarget_stability
input_exit
director_tiers
director_lifecycle
interest_happiness
director_gaps
director_action_priority
discovery
world_log
event_suite
event_live_pipelines
chronicle_smoke
activity_idle_smoke
activity_status_smoke
activity_happiness_smoke
)
scenario=""
raw_cmd=""
while [[ $# -gt 0 ]]; do
case "$1" in
--cmd)
raw_cmd="$2"
shift 2
;;
--timeout)
TIMEOUT_SEC="$2"
shift 2
;;
--repeat)
REPEAT="$2"
shift 2
;;
--no-launch)
DO_LAUNCH=0
shift
;;
-*)
echo "Unknown flag: $1" >&2
exit 2
;;
*)
scenario="$1"
shift
;;
esac
done
if [[ -z "$scenario" && -z "$raw_cmd" ]]; then
scenario="critical_smoke"
fi
worldbox_running() {
local pid cmd
for pid in $(pgrep -a worldbox 2>/dev/null | awk '{print $1}'); do
cmd=$(tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null || true)
if [[ "$cmd" == *"/common/worldbox/worldbox"* && "$cmd" != *zsh* && "$cmd" != *snap=* && "$cmd" != *harness-run* ]]; then
return 0
fi
done
return 1
}
mod_compile_failed() {
[[ -f "$LOG" ]] || return 1
# Current Player.log session only (Unity truncates the file on each boot).
rg -q "error CS" "$LOG" 2>/dev/null || return 1
# Only fail when IdleSpectator is implicated, or NML reports a mod compile error near CS lines.
if rg -q "IdleSpectator.*error CS|error CS.*IdleSpectator|WatchCaption\.cs|Chronicle\.cs|LoreProse\.cs|AgentHarness\.cs|ChronicleHud\.cs" "$LOG" 2>/dev/null; then
return 0
fi
# Broader: any error CS in the same boot as an IdleSpectator compile attempt.
if rg -q "Compile Mod IdleSpectator" "$LOG" 2>/dev/null && rg -q "error CS" "$LOG" 2>/dev/null; then
return 0
fi
return 1
}
mod_ready() {
[[ -f "$LOG" ]] || return 1
if mod_compile_failed; then
return 2
fi
rg -q "\[IdleSpectator\]: Harmony patches applied" "$LOG" 2>/dev/null
}
report_compile_errors() {
echo "IdleSpectator compile failed:" >&2
if [[ -f "$LOG" ]]; then
rg -n "error CS|Compile Mod IdleSpectator|Failed to compile|WatchCaption\.cs|AgentHarness\.cs" "$LOG" | tail -n 50 >&2 || true
fi
}
ensure_worldbox() {
if worldbox_running; then
echo "WorldBox already running"
local ready_rc=0
mod_ready || ready_rc=$?
if (( ready_rc == 2 )); then
report_compile_errors
return 1
fi
if (( ready_rc != 0 )); then
echo "WARNING: IdleSpectator harmony ready-line not seen in log (continuing)" >&2
fi
return 0
fi
if (( DO_LAUNCH == 0 )); then
echo "WorldBox is not running (pass without --no-launch to auto-start)" >&2
return 1
fi
# Do NOT write a marker into Player.log - Unity truncates that file on boot.
echo "Starting WorldBox (steam app $STEAM_APP_ID) ..."
echo "===== HARNESS LAUNCH $(date -Iseconds) =====" >> /tmp/idle-spectator-harness-launch.log 2>/dev/null || true
nohup steam -applaunch "$STEAM_APP_ID" >/tmp/idle-spectator-harness-launch.log 2>&1 &
disown || true
local deadline=$((SECONDS + BOOT_TIMEOUT_SEC))
local saw_process=0
while (( SECONDS < deadline )); do
if worldbox_running; then
saw_process=1
printf '.'
else
printf 'o'
fi
local ready_rc=0
mod_ready || ready_rc=$?
if (( ready_rc == 0 )); then
# Avoid accepting a stale log from a previous boot before the new process exists.
if (( saw_process == 1 )) || worldbox_running; then
echo
echo "WorldBox + IdleSpectator ready"
return 0
fi
fi
if (( ready_rc == 2 )); then
echo
report_compile_errors
return 1
fi
sleep 2
done
echo >&2
echo "Timed out waiting for WorldBox/IdleSpectator boot (${BOOT_TIMEOUT_SEC}s)" >&2
tail -n 40 "$LOG" 2>/dev/null >&2 || true
return 1
}
queue_and_wait() {
local run_label="$1"
local scenario_name="$2"
rm -f "$HARNESS/last-result.json"
: > "$HARNESS/results.jsonl"
echo 0 > "$HARNESS/offset"
: > "$HARNESS/commands.jsonl"
touch "$HARNESS/reset"
touch "$HARNESS/auto-dismiss"
sleep 0.35
{
echo '{"id":"batch","action":"reset_counters","value":"all"}'
echo '{"id":"dismiss","action":"dismiss_windows"}'
if [[ -n "$raw_cmd" ]]; then
echo "$raw_cmd"
else
echo "{\"id\":\"run\",\"action\":\"scenario\",\"value\":\"$scenario_name\"}"
fi
} > "$HARNESS/commands.jsonl"
echo "Harness queued: ${scenario_name:-raw} ($run_label) (mod=$MOD)"
echo "Waiting up to ${TIMEOUT_SEC}s for last-result.json (load a world if at the menu) ..."
local deadline=$((SECONDS + TIMEOUT_SEC))
while (( SECONDS < deadline )); do
if [[ -f "$HARNESS/last-result.json" ]]; then
# NOTE: do not name this `status` - zsh treats that as read-only.
local result_status
result_status="$(python3 -c "import json;print(json.load(open('$HARNESS/last-result.json')).get('status',''))" 2>/dev/null || true)"
if [[ "$result_status" == "PASS" || "$result_status" == "FAIL" || "$result_status" == "error" ]]; then
echo "===== last-result.json ($run_label) ====="
cat "$HARNESS/last-result.json"
echo "===== recent harness log ====="
if [[ -f "$LOG" ]]; then
rg -n "\[IdleSpectator\]\[(HARNESS|ASSERT|SNAP)\]|dismissed window|retry id=" "$LOG" | tail -n 50 || true
fi
if [[ "$result_status" == "PASS" ]]; then
# Validate Unity-written dossier crops (exact UI rect), not 2880p full frames.
sleep 1.0
mapfile -t dossier_crops < <(compgen -G "$HARNESS/hud-*-dossier.png" || true)
if ((${#dossier_crops[@]} > 0)); then
python3 "$ROOT/scripts/crop-dossier-hud.py" --strict "${dossier_crops[@]}" \
| tee /tmp/idle-dossier-crop.txt || true
else
echo "NOTE: no hud-*-dossier.png yet (need mod build with in-engine crop)" >&2
fi
return 0
fi
return 1
fi
fi
sleep 0.5
done
echo "Timed out after ${TIMEOUT_SEC}s ($run_label)" >&2
if [[ -f "$HARNESS/last-result.json" ]]; then
cat "$HARNESS/last-result.json" >&2 || true
fi
if [[ -f "$LOG" ]]; then
rg -n "IdleSpectator|error CS|Exception" "$LOG" | tail -n 50 >&2 || true
fi
return 1
}
run_one_scenario() {
local name="$1"
local label="$2"
local i pass_count=0
for ((i=1; i<=REPEAT; i++)); do
echo "======== ${name} RUN $i / $REPEAT ($label) ========"
if queue_and_wait "${name} $i/$REPEAT" "$name"; then
pass_count=$((pass_count + 1))
echo "RUN $i PASS ($name)"
else
echo "RUN $i FAIL ($name)" >&2
echo "Summary: $pass_count / $REPEAT passed for $name before failure" >&2
return 1
fi
done
echo "Summary: $pass_count / $REPEAT PASS ($name)"
return 0
}
mkdir -p "$HARNESS"
ensure_worldbox
if (( TIMEOUT_SEC < 120 )); then
TIMEOUT_SEC=120
fi
if [[ "$scenario" == "regression" || "$scenario" == "full" ]]; then
# Director/discovery waits need headroom when the suite runs many scenarios.
if (( TIMEOUT_SEC < 180 )); then
TIMEOUT_SEC=180
fi
suite_pass=0
suite_total=${#REGRESSION_SCENARIOS[@]}
echo "======== REGRESSION SUITE ($suite_total scenarios, repeat=$REPEAT) ========"
for name in "${REGRESSION_SCENARIOS[@]}"; do
if run_one_scenario "$name" "regression"; then
suite_pass=$((suite_pass + 1))
else
echo "REGRESSION FAIL at scenario=$name ($suite_pass / $suite_total completed)" >&2
exit 1
fi
done
echo "REGRESSION Summary: $suite_pass / $suite_total scenarios PASS (repeat=$REPEAT each)"
exit 0
fi
run_one_scenario "${scenario:-raw}" "single"
exit $?
Reference in a new issue View git blame Copy permalink