297 lines
8.7 KiB
Bash
Executable file
297 lines
8.7 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Run an IdleSpectator agent harness scenario (or raw JSONL commands).
|
|
# Starts WorldBox via Steam when it is not already running.
|
|
#
|
|
# Usage:
|
|
# ./scripts/harness-run.sh # default: critical_smoke
|
|
# ./scripts/harness-run.sh critical_smoke
|
|
# ./scripts/harness-run.sh regression # full suite (named scenarios)
|
|
# ./scripts/harness-run.sh --repeat 3 # run scenario 3 times (all must PASS)
|
|
# ./scripts/harness-run.sh tip_match
|
|
# ./scripts/harness-run.sh smoke
|
|
# ./scripts/harness-run.sh --cmd '{"id":"1","action":"snapshot"}'
|
|
# ./scripts/harness-run.sh --no-launch smoke
|
|
set -euo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
|
MOD="${IDLE_SPECTATOR_MOD:-$ROOT/IdleSpectator}"
|
|
HARNESS="$MOD/.harness"
|
|
LOG="${WORLDBOX_LOG:-$HOME/.config/unity3d/mkarpenko/WorldBox/Player.log}"
|
|
TIMEOUT_SEC="${HARNESS_TIMEOUT:-90}"
|
|
STEAM_APP_ID="${WORLDBOX_APP_ID:-1206560}"
|
|
BOOT_TIMEOUT_SEC="${HARNESS_BOOT_TIMEOUT:-180}"
|
|
DO_LAUNCH=1
|
|
REPEAT=1
|
|
|
|
# Full regression gate (battle / map-gen intentionally excluded).
|
|
REGRESSION_SCENARIOS=(
|
|
critical_smoke
|
|
settings_full
|
|
ghost_guard
|
|
retarget_stability
|
|
input_exit
|
|
director_tiers
|
|
director_lifecycle
|
|
interest_happiness
|
|
director_gaps
|
|
director_action_priority
|
|
discovery
|
|
world_log
|
|
camera_presentability
|
|
event_suite
|
|
chronicle_smoke
|
|
activity_idle_smoke
|
|
activity_status_smoke
|
|
activity_happiness_smoke
|
|
# Heavier live fires last - wars/meta/eras can reshape the loaded world.
|
|
event_live_pipelines
|
|
)
|
|
|
|
scenario=""
|
|
raw_cmd=""
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--cmd)
|
|
raw_cmd="$2"
|
|
shift 2
|
|
;;
|
|
--timeout)
|
|
TIMEOUT_SEC="$2"
|
|
shift 2
|
|
;;
|
|
--repeat)
|
|
REPEAT="$2"
|
|
shift 2
|
|
;;
|
|
--no-launch)
|
|
DO_LAUNCH=0
|
|
shift
|
|
;;
|
|
-*)
|
|
echo "Unknown flag: $1" >&2
|
|
exit 2
|
|
;;
|
|
*)
|
|
scenario="$1"
|
|
shift
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [[ -z "$scenario" && -z "$raw_cmd" ]]; then
|
|
scenario="critical_smoke"
|
|
fi
|
|
|
|
worldbox_running() {
|
|
local pid cmd
|
|
for pid in $(pgrep -a worldbox 2>/dev/null | awk '{print $1}'); do
|
|
cmd=$(tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null || true)
|
|
if [[ "$cmd" == *"/common/worldbox/worldbox"* && "$cmd" != *zsh* && "$cmd" != *snap=* && "$cmd" != *harness-run* ]]; then
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
mod_compile_failed() {
|
|
[[ -f "$LOG" ]] || return 1
|
|
# Current Player.log session only (Unity truncates the file on each boot).
|
|
rg -q "error CS" "$LOG" 2>/dev/null || return 1
|
|
# Only fail when IdleSpectator is implicated, or NML reports a mod compile error near CS lines.
|
|
if rg -q "IdleSpectator.*error CS|error CS.*IdleSpectator|WatchCaption\.cs|Chronicle\.cs|LoreProse\.cs|AgentHarness\.cs|ChronicleHud\.cs" "$LOG" 2>/dev/null; then
|
|
return 0
|
|
fi
|
|
# Broader: any error CS in the same boot as an IdleSpectator compile attempt.
|
|
if rg -q "Compile Mod IdleSpectator" "$LOG" 2>/dev/null && rg -q "error CS" "$LOG" 2>/dev/null; then
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
mod_ready() {
|
|
[[ -f "$LOG" ]] || return 1
|
|
if mod_compile_failed; then
|
|
return 2
|
|
fi
|
|
rg -q "\[IdleSpectator\]: Harmony patches applied" "$LOG" 2>/dev/null
|
|
}
|
|
|
|
report_compile_errors() {
|
|
echo "IdleSpectator compile failed:" >&2
|
|
if [[ -f "$LOG" ]]; then
|
|
rg -n "error CS|Compile Mod IdleSpectator|Failed to compile|WatchCaption\.cs|AgentHarness\.cs" "$LOG" | tail -n 50 >&2 || true
|
|
fi
|
|
}
|
|
|
|
ensure_worldbox() {
|
|
if worldbox_running; then
|
|
echo "WorldBox already running"
|
|
local ready_rc=0
|
|
mod_ready || ready_rc=$?
|
|
if (( ready_rc == 2 )); then
|
|
report_compile_errors
|
|
return 1
|
|
fi
|
|
if (( ready_rc != 0 )); then
|
|
echo "WARNING: IdleSpectator harmony ready-line not seen in log (continuing)" >&2
|
|
fi
|
|
return 0
|
|
fi
|
|
|
|
if (( DO_LAUNCH == 0 )); then
|
|
echo "WorldBox is not running (pass without --no-launch to auto-start)" >&2
|
|
return 1
|
|
fi
|
|
|
|
# Do NOT write a marker into Player.log - Unity truncates that file on boot.
|
|
echo "Starting WorldBox (steam app $STEAM_APP_ID) ..."
|
|
echo "===== HARNESS LAUNCH $(date -Iseconds) =====" >> /tmp/idle-spectator-harness-launch.log 2>/dev/null || true
|
|
nohup steam -applaunch "$STEAM_APP_ID" >/tmp/idle-spectator-harness-launch.log 2>&1 &
|
|
disown || true
|
|
|
|
local deadline=$((SECONDS + BOOT_TIMEOUT_SEC))
|
|
local saw_process=0
|
|
while (( SECONDS < deadline )); do
|
|
if worldbox_running; then
|
|
saw_process=1
|
|
printf '.'
|
|
else
|
|
printf 'o'
|
|
fi
|
|
|
|
local ready_rc=0
|
|
mod_ready || ready_rc=$?
|
|
if (( ready_rc == 0 )); then
|
|
# Avoid accepting a stale log from a previous boot before the new process exists.
|
|
if (( saw_process == 1 )) || worldbox_running; then
|
|
echo
|
|
echo "WorldBox + IdleSpectator ready"
|
|
return 0
|
|
fi
|
|
fi
|
|
if (( ready_rc == 2 )); then
|
|
echo
|
|
report_compile_errors
|
|
return 1
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo >&2
|
|
echo "Timed out waiting for WorldBox/IdleSpectator boot (${BOOT_TIMEOUT_SEC}s)" >&2
|
|
tail -n 40 "$LOG" 2>/dev/null >&2 || true
|
|
return 1
|
|
}
|
|
|
|
queue_and_wait() {
|
|
local run_label="$1"
|
|
local scenario_name="$2"
|
|
rm -f "$HARNESS/last-result.json"
|
|
: > "$HARNESS/results.jsonl"
|
|
echo 0 > "$HARNESS/offset"
|
|
: > "$HARNESS/commands.jsonl"
|
|
touch "$HARNESS/reset"
|
|
touch "$HARNESS/auto-dismiss"
|
|
sleep 0.35
|
|
|
|
{
|
|
echo '{"id":"batch","action":"reset_counters","value":"all"}'
|
|
echo '{"id":"dismiss","action":"dismiss_windows"}'
|
|
if [[ -n "$raw_cmd" ]]; then
|
|
echo "$raw_cmd"
|
|
else
|
|
echo "{\"id\":\"run\",\"action\":\"scenario\",\"value\":\"$scenario_name\"}"
|
|
fi
|
|
} > "$HARNESS/commands.jsonl"
|
|
|
|
echo "Harness queued: ${scenario_name:-raw} ($run_label) (mod=$MOD)"
|
|
echo "Waiting up to ${TIMEOUT_SEC}s for last-result.json (load a world if at the menu) ..."
|
|
|
|
local deadline=$((SECONDS + TIMEOUT_SEC))
|
|
while (( SECONDS < deadline )); do
|
|
if [[ -f "$HARNESS/last-result.json" ]]; then
|
|
# NOTE: do not name this `status` - zsh treats that as read-only.
|
|
local result_status
|
|
result_status="$(python3 -c "import json;print(json.load(open('$HARNESS/last-result.json')).get('status',''))" 2>/dev/null || true)"
|
|
if [[ "$result_status" == "PASS" || "$result_status" == "FAIL" || "$result_status" == "error" ]]; then
|
|
echo "===== last-result.json ($run_label) ====="
|
|
cat "$HARNESS/last-result.json"
|
|
echo "===== recent harness log ====="
|
|
if [[ -f "$LOG" ]]; then
|
|
rg -n "\[IdleSpectator\]\[(HARNESS|ASSERT|SNAP)\]|dismissed window|retry id=" "$LOG" | tail -n 50 || true
|
|
fi
|
|
if [[ "$result_status" == "PASS" ]]; then
|
|
# Validate Unity-written dossier crops (exact UI rect), not 2880p full frames.
|
|
sleep 1.0
|
|
mapfile -t dossier_crops < <(compgen -G "$HARNESS/hud-*-dossier.png" || true)
|
|
if ((${#dossier_crops[@]} > 0)); then
|
|
python3 "$ROOT/scripts/crop-dossier-hud.py" --strict "${dossier_crops[@]}" \
|
|
| tee /tmp/idle-dossier-crop.txt || true
|
|
else
|
|
echo "NOTE: no hud-*-dossier.png yet (need mod build with in-engine crop)" >&2
|
|
fi
|
|
return 0
|
|
fi
|
|
return 1
|
|
fi
|
|
fi
|
|
sleep 0.5
|
|
done
|
|
|
|
echo "Timed out after ${TIMEOUT_SEC}s ($run_label)" >&2
|
|
if [[ -f "$HARNESS/last-result.json" ]]; then
|
|
cat "$HARNESS/last-result.json" >&2 || true
|
|
fi
|
|
if [[ -f "$LOG" ]]; then
|
|
rg -n "IdleSpectator|error CS|Exception" "$LOG" | tail -n 50 >&2 || true
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
run_one_scenario() {
|
|
local name="$1"
|
|
local label="$2"
|
|
local i pass_count=0
|
|
for ((i=1; i<=REPEAT; i++)); do
|
|
echo "======== ${name} RUN $i / $REPEAT ($label) ========"
|
|
if queue_and_wait "${name} $i/$REPEAT" "$name"; then
|
|
pass_count=$((pass_count + 1))
|
|
echo "RUN $i PASS ($name)"
|
|
else
|
|
echo "RUN $i FAIL ($name)" >&2
|
|
echo "Summary: $pass_count / $REPEAT passed for $name before failure" >&2
|
|
return 1
|
|
fi
|
|
done
|
|
echo "Summary: $pass_count / $REPEAT PASS ($name)"
|
|
return 0
|
|
}
|
|
|
|
mkdir -p "$HARNESS"
|
|
ensure_worldbox
|
|
|
|
if (( TIMEOUT_SEC < 120 )); then
|
|
TIMEOUT_SEC=120
|
|
fi
|
|
|
|
if [[ "$scenario" == "regression" || "$scenario" == "full" ]]; then
|
|
# Director/discovery waits need headroom when the suite runs many scenarios.
|
|
if (( TIMEOUT_SEC < 180 )); then
|
|
TIMEOUT_SEC=180
|
|
fi
|
|
suite_pass=0
|
|
suite_total=${#REGRESSION_SCENARIOS[@]}
|
|
echo "======== REGRESSION SUITE ($suite_total scenarios, repeat=$REPEAT) ========"
|
|
for name in "${REGRESSION_SCENARIOS[@]}"; do
|
|
if run_one_scenario "$name" "regression"; then
|
|
suite_pass=$((suite_pass + 1))
|
|
else
|
|
echo "REGRESSION FAIL at scenario=$name ($suite_pass / $suite_total completed)" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
echo "REGRESSION Summary: $suite_pass / $suite_total scenarios PASS (repeat=$REPEAT each)"
|
|
exit 0
|
|
fi
|
|
|
|
run_one_scenario "${scenario:-raw}" "single"
|
|
exit $?
|