#!/usr/bin/env bash
set -euo pipefail

ROOT="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
RUNTIME_ROOT="${Q4_REPLAY_RUNTIME_ROOT:-${ROOT}/runtime/native}"
OUT_ROOT="${ROOT}/runtime/native-smoke/repeater"
DEMO_SELECTOR="${ROOT}/port/native/select-server-demo.sh"
PORT="${Q4_REPLAY_SMOKE_PORT:-27667}"
TIMEOUT_SECONDS="${Q4_REPLAY_SMOKE_TIMEOUT:-120}"
EXIT_TIMEOUT_SECONDS="${Q4_REPLAY_EXIT_TIMEOUT:-20}"
FIRST_VIEWER_ADVANCE_MS=5000
POST_JOIN_OBSERVE_SECONDS=5
POST_JOIN_MIN_ADVANCE_MS=3000
SYNC_TOLERANCE_MS=1000
MIN_POST_JOIN_SNAPSHOTS=2
MIN_DEMO_DURATION_MS=60000
SERVER_RAW="${OUT_ROOT}/server.raw.log"
SERVER_LOG="${OUT_ROOT}/server.log"
VIEWER1_RAW="${OUT_ROOT}/viewer1.raw.log"
VIEWER1_LOG="${OUT_ROOT}/viewer1.log"
VIEWER2_RAW="${OUT_ROOT}/viewer2.raw.log"
VIEWER2_LOG="${OUT_ROOT}/viewer2.log"
FAILURE_PATTERN='COMMAND_EXIT_CODE="[1-9][0-9]*"|Fatal Error|fatal signal|free\(\): invalid pointer|segmentation fault|core dumped|reliable messages overflow|snapshot .* contains too many|couldn.t apply snapshot|viewer [0-9]+ connected \(repeater\)\.'

if [[ $# -gt 1 ]]; then
    echo "Usage: $0 [demo.q4sd]" >&2
    exit 2
fi

if [[ $# -eq 1 ]]; then
    DEMO_FILE="$(realpath -- "$1")"
else
    # Late join needs enough recorded time for two clients to enter. A map
    # rotation can leave a short trailing demo. Prefer the largest complete
    # smoke recording; consult legacy production recordings only when the
    # smoke directory contains no complete demo at all.
    DEMO_FILE="$(Q4_DEMO_MIN_DURATION_MS="${MIN_DEMO_DURATION_MS}" "${DEMO_SELECTOR}" \
        "${ROOT}/runtime/native-smoke/replays" \
        "${ROOT}/replays" 2>/dev/null)" || {
        echo "No server demo found; run port/native/smoke-dedicated.sh first" >&2
        exit 1
    }
fi

[[ -x "${RUNTIME_ROOT}/openQ4-ded_x64" && -x "${RUNTIME_ROOT}/openQ4-client_x64" ]] || {
    echo "Native client/server binaries are missing; run tools/build-native.sh first" >&2
    exit 1
}
[[ -f "${DEMO_FILE}" && "${DEMO_FILE}" == *.q4sd ]] || {
    echo "Server demo is missing or has the wrong extension: ${DEMO_FILE}" >&2
    exit 1
}
for command in awk pgrep ps script setsid ss xvfb-run; do
    command -v "${command}" >/dev/null 2>&1 || {
        echo "Required replay smoke command is missing: ${command}" >&2
        exit 1
    }
done
if ss -H -lun "sport = :${PORT}" | grep -q .; then
    echo "UDP port ${PORT} is already in use" >&2
    exit 1
fi

mkdir -p -- "${OUT_ROOT}"
rm -rf -- "${OUT_ROOT}/server-home" "${OUT_ROOT}/viewer1-home" "${OUT_ROOT}/viewer2-home"
mkdir -p -- "${OUT_ROOT}/server-home" "${OUT_ROOT}/viewer1-home" "${OUT_ROOT}/viewer2-home"
rm -f -- "${SERVER_RAW}" "${SERVER_LOG}" "${VIEWER1_RAW}" "${VIEWER1_LOG}" \
    "${VIEWER2_RAW}" "${VIEWER2_LOG}" "${OUT_ROOT}"/*.in
mkfifo -- "${OUT_ROOT}/server.in" "${OUT_ROOT}/viewer1.in" "${OUT_ROOT}/viewer2.in"
exec 3<>"${OUT_ROOT}/server.in"
exec 4<>"${OUT_ROOT}/viewer1.in"
exec 5<>"${OUT_ROOT}/viewer2.in"

server_pid=""
viewer1_pid=""
viewer2_pid=""

collect_tree() {
    local pid="$1"
    local child
    while read -r child; do
        [[ -n "${child}" ]] && collect_tree "${child}"
    done < <(pgrep -P "${pid}" 2>/dev/null || true)
    printf '%s\n' "${pid}"
}

stop_tree() {
    local pid="$1"
    local -a pids=()
    [[ -n "${pid}" ]] || return 0
    mapfile -t pids < <(collect_tree "${pid}")
    kill -TERM -- "-${pid}" 2>/dev/null || true
    kill -TERM -- "${pids[@]}" 2>/dev/null || true
    for (( i = 0; i < 20; ++i )); do
        local alive=0
        local member
        for member in "${pids[@]}"; do
            kill -0 "${member}" 2>/dev/null && alive=1
        done
        (( alive == 0 )) && return
        sleep 0.1
    done
    kill -KILL -- "-${pid}" 2>/dev/null || true
    kill -KILL -- "${pids[@]}" 2>/dev/null || true
}

stop_by_home() {
    local -a pids=()
    mapfile -t pids < <(pgrep -f -- "${OUT_ROOT}/(server-home|viewer1-home|viewer2-home)" 2>/dev/null || true)
    [[ ${#pids[@]} -eq 0 ]] || kill -TERM -- "${pids[@]}" 2>/dev/null || true
    sleep 0.2
    [[ ${#pids[@]} -eq 0 ]] || kill -KILL -- "${pids[@]}" 2>/dev/null || true
}

cleanup() {
    set +e
    stop_tree "${viewer2_pid}"
    stop_tree "${viewer1_pid}"
    stop_tree "${server_pid}"
    stop_by_home
    wait "${viewer2_pid}" "${viewer1_pid}" "${server_pid}" 2>/dev/null || true
    exec 5>&-
    exec 4>&-
    exec 3>&-
    rm -f -- "${OUT_ROOT}"/*.in
}
trap 'exit 130' INT
trap 'exit 143' TERM
trap cleanup EXIT

snapshot_rows() {
    local log="$1"
    awk '
        index($0, "received first snapshot,") || index($0, "received snapshot,") {
            init = $0
            sub(/^.*gameInitId = /, "", init)
            sub(/,.*/, "", init)
            time = $0
            sub(/^.*gameTime( =)? /, "", time)
            sub(/[^0-9-].*$/, "", time)
            if (init ~ /^-?[0-9]+$/ && time ~ /^-?[0-9]+$/) {
                print init, time
            }
        }
    ' "${log}" 2>/dev/null
}

snapshot_count() {
    snapshot_rows "$1" | awk 'END { print NR + 0 }'
}

snapshot_delta() {
    local left="$1"
    local right="$2"
    local delta=$(( left - right ))
    (( delta < 0 )) && delta=$(( -delta ))
    printf '%d\n' "${delta}"
}

assert_process_healthy() {
    local pid="$1"
    local label="$2"
    local log="$3"
    local state=""

    if ! kill -0 "${pid}" 2>/dev/null; then
        echo "${label} exited during replay observation" >&2
        return 1
    fi
    state="$(ps -o stat= -p "${pid}" 2>/dev/null || true)"
    if [[ "${state}" == Z* ]] || grep -Eq 'COMMAND_EXIT_CODE="[0-9]+"' "${log}" 2>/dev/null; then
        echo "${label} stopped during replay observation" >&2
        return 1
    fi
    if grep -Eiq "${FAILURE_PATTERN}" "${log}" 2>/dev/null; then
        echo "${label} logged a runtime, protocol, or repeater-only failure" >&2
        return 1
    fi
}

observe_live_replay() {
    local duration="$1"
    local started=${SECONDS}

    while (( SECONDS - started < duration )); do
        assert_process_healthy "${server_pid}" "replay server" "${SERVER_RAW}"
        assert_process_healthy "${viewer1_pid}" "first replay viewer" "${VIEWER1_RAW}"
        assert_process_healthy "${viewer2_pid}" "late replay viewer" "${VIEWER2_RAW}"
        if grep -Fq 'finished network server demo' "${SERVER_RAW}" 2>/dev/null; then
            echo "Replay ended during the two-viewer observation window" >&2
            return 1
        fi
        sleep 0.25
    done
}

wait_for_log() {
    local pid="$1"
    local log="$2"
    local pattern="$3"
    local timeout="$4"
    local guard_pid="${5:-}"
    local guard_log="${6:-}"
    local started=${SECONDS}
    until grep -Fq -- "${pattern}" "${log}" 2>/dev/null; do
        if grep -Eiq "${FAILURE_PATTERN}" "${log}" 2>/dev/null; then
            echo "Process logged a runtime, protocol, or repeater-only failure while waiting for: ${pattern}" >&2
            return 1
        fi
        if grep -Eq 'COMMAND_EXIT_CODE="[0-9]+"' "${log}" 2>/dev/null; then
            echo "Process exited before producing: ${pattern}" >&2
            return 1
        fi
        if ! kill -0 "${pid}" 2>/dev/null; then
            echo "Process exited while waiting for: ${pattern}" >&2
            return 1
        fi
        if [[ -n "${guard_pid}" ]] && ! kill -0 "${guard_pid}" 2>/dev/null; then
            echo "Guard process exited while waiting for: ${pattern}" >&2
            return 1
        fi
        if [[ -n "${guard_log}" ]] && grep -Eq 'COMMAND_EXIT_CODE="[0-9]+"' "${guard_log}" 2>/dev/null; then
            echo "Guard process exited before producing: ${pattern}" >&2
            return 1
        fi
        if [[ -n "${guard_log}" ]] && grep -Eiq "${FAILURE_PATTERN}" "${guard_log}" 2>/dev/null; then
            echo "Guard process logged a runtime, protocol, or repeater-only failure while waiting for: ${pattern}" >&2
            return 1
        fi
        if (( SECONDS - started >= timeout )); then
            echo "Timed out waiting for replay evidence: ${pattern}" >&2
            return 1
        fi
        sleep 0.25
    done
}

wait_for_snapshot_at_least() {
    local pid="$1"
    local log="$2"
    local expected_init="$3"
    local minimum_time="$4"
    local timeout="$5"
    local guard_pid="$6"
    local guard_log="$7"
    local started=${SECONDS}
    local tuple=""
    local init=""
    local game_time=""

    while true; do
        tuple="$(snapshot_rows "${log}" | tail -n 1)"
        if [[ -n "${tuple}" ]]; then
            read -r init game_time <<<"${tuple}"
            if [[ "${init}" != "${expected_init}" ]]; then
                echo "Snapshot gameInitId changed while waiting for ${minimum_time}: ${expected_init} -> ${init}" >&2
                return 1
            fi
            if [[ "${game_time}" =~ ^-?[0-9]+$ ]] && (( game_time >= minimum_time )); then
                printf '%s %s\n' "${init}" "${game_time}"
                return 0
            fi
        fi
        assert_process_healthy "${pid}" "snapshot client" "${log}"
        assert_process_healthy "${guard_pid}" "snapshot server" "${guard_log}"
        if (( SECONDS - started >= timeout )); then
            echo "Timed out waiting for snapshot time ${minimum_time}" >&2
            return 1
        fi
        sleep 0.25
    done
}

wait_for_clean_exit() {
    local pid="$1"
    local label="$2"
    local started=${SECONDS}
    local state=""
    local status=0

    while kill -0 "${pid}" 2>/dev/null; do
        state="$(ps -o stat= -p "${pid}" 2>/dev/null || true)"
        [[ "${state}" == Z* ]] && break
        if (( SECONDS - started >= EXIT_TIMEOUT_SECONDS )); then
            echo "${label} did not exit within ${EXIT_TIMEOUT_SECONDS}s" >&2
            return 1
        fi
        sleep 0.25
    done
    if wait "${pid}"; then
        return 0
    else
        status=$?
        echo "${label} exited with status ${status}" >&2
        return 1
    fi
}

server_command=(
    "${RUNTIME_ROOT}/openQ4-ded_x64"
    +set fs_basepath "${RUNTIME_ROOT}"
    +set fs_savepath "${OUT_ROOT}/server-home"
    +set net_ip 127.0.0.1
    +set net_port "${PORT}"
    +set net_LANServer 1
    +set net_serverDemo 0
    +set timescale 1
    +set si_gameType DM
    +set si_pure 0
    +set net_serverAllowDataChecksumMismatch 1
    +set g_repeaterReliableOnly 0
    +set ri_maxViewers 4
    +set ri_waitForViewer 1
    +playServerDemoServer "${DEMO_FILE}"
)
setsid "${server_command[@]}" <&3 >"${SERVER_RAW}" 2>&1 &
server_pid=$!

wait_for_log "${server_pid}" "${SERVER_RAW}" "serving full-state server demo ${DEMO_FILE} on port ${PORT}" "${TIMEOUT_SECONDS}"
sleep 3
if grep -Fq 'starting network server demo' "${SERVER_RAW}" || grep -Fq 'finished network server demo' "${SERVER_RAW}"; then
    echo "Replay advanced before the first viewer entered the game" >&2
    exit 1
fi

viewer1_command=(
    "${RUNTIME_ROOT}/openQ4-client_x64"
    +set fs_basepath "${RUNTIME_ROOT}"
    +set fs_savepath "${OUT_ROOT}/viewer1-home"
    +set com_skipLogoVideos 1
    +set com_skipRenderer 1
    +set r_fullscreen 0
    +set r_mode 3
    +set s_noSound 1
    +set sys_allowMultipleInstances 1
    +set net_LANServer 1
    +set net_verbose 2
    +set si_gameType DM
    +set ui_name ReplayViewer1
    +connect "127.0.0.1:${PORT}"
)
printf -v viewer1_command_line '%q ' "${viewer1_command[@]}"
ALSOFT_DRIVERS=null TERM=xterm setsid xvfb-run -a -s '-screen 0 1280x720x24' \
    script -qefc "${viewer1_command_line}" "${VIEWER1_RAW}" <&4 >/dev/null 2>&1 &
viewer1_pid=$!

wait_for_log "${server_pid}" "${SERVER_RAW}" 'viewer 0 connected.' "${TIMEOUT_SECONDS}" "${viewer1_pid}" "${VIEWER1_RAW}"
wait_for_log "${viewer1_pid}" "${VIEWER1_RAW}" 'received first snapshot' "${TIMEOUT_SECONDS}" "${server_pid}" "${SERVER_RAW}"
wait_for_log "${server_pid}" "${SERVER_RAW}" 'starting network server demo' "${TIMEOUT_SECONDS}"
first_tuple="$(snapshot_rows "${VIEWER1_RAW}" | head -n 1)"
read -r first_init first_time <<<"${first_tuple}"
if [[ ! "${first_init}" =~ ^-?[0-9]+$ || ! "${first_time}" =~ ^-?[0-9]+$ ]]; then
    echo "Could not read the first viewer snapshot identity and time" >&2
    exit 1
fi
first_snapshot_wall=${SECONDS}

prejoin_tuple="$(wait_for_snapshot_at_least \
    "${viewer1_pid}" "${VIEWER1_RAW}" "${first_init}" \
    "$(( first_time + FIRST_VIEWER_ADVANCE_MS ))" "${TIMEOUT_SECONDS}" \
    "${server_pid}" "${SERVER_RAW}")"
read -r prejoin_init prejoin_time <<<"${prejoin_tuple}"
prejoin_count="$(snapshot_count "${VIEWER1_RAW}")"
if (( prejoin_count < 2 )); then
    echo "First viewer did not receive multiple snapshots before late join (${prejoin_count})" >&2
    exit 1
fi
if grep -Fq 'finished network server demo' "${SERVER_RAW}"; then
    echo "Replay ended before the late viewer could join" >&2
    exit 1
fi

viewer2_command=(
    "${RUNTIME_ROOT}/openQ4-client_x64"
    +set fs_basepath "${RUNTIME_ROOT}"
    +set fs_savepath "${OUT_ROOT}/viewer2-home"
    +set com_skipLogoVideos 1
    +set com_skipRenderer 1
    +set r_fullscreen 0
    +set r_mode 3
    +set s_noSound 1
    +set sys_allowMultipleInstances 1
    +set net_LANServer 1
    +set net_verbose 2
    +set si_gameType DM
    +set ui_name ReplayViewer2
    +connect "127.0.0.1:${PORT}"
)
printf -v viewer2_command_line '%q ' "${viewer2_command[@]}"
viewer2_started_wall=${SECONDS}
ALSOFT_DRIVERS=null TERM=xterm setsid xvfb-run -a -s '-screen 0 1280x720x24' \
    script -qefc "${viewer2_command_line}" "${VIEWER2_RAW}" <&5 >/dev/null 2>&1 &
viewer2_pid=$!

wait_for_log "${server_pid}" "${SERVER_RAW}" 'viewer 1 connected.' "${TIMEOUT_SECONDS}" "${viewer2_pid}" "${VIEWER2_RAW}"
wait_for_log "${viewer2_pid}" "${VIEWER2_RAW}" 'received first snapshot' "${TIMEOUT_SECONDS}" "${server_pid}" "${SERVER_RAW}"
second_tuple="$(snapshot_rows "${VIEWER2_RAW}" | head -n 1)"
read -r second_init second_time <<<"${second_tuple}"
if [[ ! "${second_init}" =~ ^-?[0-9]+$ || ! "${second_time}" =~ ^-?[0-9]+$ ]]; then
    echo "Could not read the late viewer snapshot identity and time" >&2
    exit 1
fi
viewer1_join_tuple="$(snapshot_rows "${VIEWER1_RAW}" | tail -n 1)"
read -r viewer1_join_init viewer1_join_time <<<"${viewer1_join_tuple}"
if [[ "${second_init}" != "${first_init}" || "${viewer1_join_init}" != "${first_init}" ]]; then
    echo "Viewer gameInitId mismatch at late join (${first_init}, ${viewer1_join_init}, ${second_init})" >&2
    exit 1
fi
join_delta="$(snapshot_delta "${viewer1_join_time}" "${second_time}")"
if (( join_delta > SYNC_TOLERANCE_MS )); then
    echo "Late viewer did not join contemporaneous replay state (${viewer1_join_time} vs ${second_time}, delta ${join_delta}ms)" >&2
    exit 1
fi

late_viewer_load_seconds=$(( SECONDS - viewer2_started_wall ))
replay_active_seconds=$(( SECONDS - first_snapshot_wall ))
viewer1_join_count="$(snapshot_count "${VIEWER1_RAW}")"
viewer2_join_count="$(snapshot_count "${VIEWER2_RAW}")"

observe_live_replay "${POST_JOIN_OBSERVE_SECONDS}"

viewer1_final_tuple="$(snapshot_rows "${VIEWER1_RAW}" | tail -n 1)"
viewer2_final_tuple="$(snapshot_rows "${VIEWER2_RAW}" | tail -n 1)"
read -r viewer1_final_init viewer1_final_time <<<"${viewer1_final_tuple}"
read -r viewer2_final_init viewer2_final_time <<<"${viewer2_final_tuple}"
if [[ "${viewer1_final_init}" != "${first_init}" || "${viewer2_final_init}" != "${first_init}" ]]; then
    echo "Viewer gameInitId changed during shared replay observation" >&2
    exit 1
fi

viewer1_advance=$(( viewer1_final_time - viewer1_join_time ))
viewer2_advance=$(( viewer2_final_time - second_time ))
if (( viewer1_advance < POST_JOIN_MIN_ADVANCE_MS || viewer2_advance < POST_JOIN_MIN_ADVANCE_MS )); then
    echo "Viewers did not both advance ${POST_JOIN_MIN_ADVANCE_MS}ms after late join (${viewer1_advance}ms, ${viewer2_advance}ms)" >&2
    exit 1
fi
final_delta="$(snapshot_delta "${viewer1_final_time}" "${viewer2_final_time}")"
if (( final_delta > SYNC_TOLERANCE_MS )); then
    echo "Viewer replay clocks diverged after shared observation (${viewer1_final_time} vs ${viewer2_final_time}, delta ${final_delta}ms)" >&2
    exit 1
fi

viewer1_final_count="$(snapshot_count "${VIEWER1_RAW}")"
viewer2_final_count="$(snapshot_count "${VIEWER2_RAW}")"
viewer1_new_snapshots=$(( viewer1_final_count - viewer1_join_count ))
viewer2_new_snapshots=$(( viewer2_final_count - viewer2_join_count ))
if (( viewer1_new_snapshots < MIN_POST_JOIN_SNAPSHOTS || viewer2_new_snapshots < MIN_POST_JOIN_SNAPSHOTS )); then
    echo "Viewers did not receive multiple post-join snapshots (${viewer1_new_snapshots}, ${viewer2_new_snapshots})" >&2
    exit 1
fi

printf 'quit\n' >&5
printf 'quit\n' >&4
printf 'quit\n' >&3

process_failure=0
if ! wait_for_clean_exit "${viewer2_pid}" "late replay viewer"; then
    process_failure=1
fi
kill -0 "${viewer2_pid}" 2>/dev/null || viewer2_pid=""
if ! wait_for_clean_exit "${viewer1_pid}" "first replay viewer"; then
    process_failure=1
fi
kill -0 "${viewer1_pid}" 2>/dev/null || viewer1_pid=""
if ! wait_for_clean_exit "${server_pid}" "replay server"; then
    process_failure=1
fi
kill -0 "${server_pid}" 2>/dev/null || server_pid=""

tr -d '\r' <"${SERVER_RAW}" >"${SERVER_LOG}"
tr -d '\r' <"${VIEWER1_RAW}" >"${VIEWER1_LOG}"
tr -d '\r' <"${VIEWER2_RAW}" >"${VIEWER2_LOG}"

for log in "${SERVER_LOG}" "${VIEWER1_LOG}" "${VIEWER2_LOG}"; do
    if grep -Eiq "${FAILURE_PATTERN}" "${log}"; then
        echo "Replay smoke log contains a runtime, protocol, or repeater-only failure: ${log}" >&2
        exit 1
    fi
done

if ss -H -lun "sport = :${PORT}" | grep -q .; then
    echo "Replay UDP port ${PORT} remained in use after shutdown" >&2
    process_failure=1
fi
if (( process_failure != 0 )); then
    exit 1
fi

echo "Replay demo: ${DEMO_FILE}"
echo "Replay server log: ${SERVER_LOG}"
echo "Replay gameInitId: ${first_init}"
echo "First viewer initial/pre-join times: ${first_time}/${prejoin_time}"
echo "Late-join viewer times: ${viewer1_join_time}/${second_time} (delta ${join_delta}ms)"
echo "Final viewer times: ${viewer1_final_time}/${viewer2_final_time} (delta ${final_delta}ms)"
echo "Post-join advances: ${viewer1_advance}ms/${viewer2_advance}ms"
echo "Post-join snapshot counts: ${viewer1_new_snapshots}/${viewer2_new_snapshots}"
echo "Late viewer load wall time: ${late_viewer_load_seconds}s"
echo "Replay active through late join: ${replay_active_seconds}s"
