#!/usr/bin/env bash
# Drive the eval against the llama-swap endpoint on the Ubuntu box.
# llama-swap auto-loads/swaps the backend by the request's "model" field, so you
# do NOT serve models manually -- just run the three tags below in sequence.
#
# Runtime reality (llama.cpp, 5090, --parallel 4, ~500 tok/s aggregate):
#   llama.cpp splits -c across slots, so concurrency is capped -- this is MUCH
#   slower than vLLM for long reasoning traces. Ballpark:
#     math  avg@16 (90 items)  : ~8h vibe + ~10h qwen3-think + ~1h base  ~= 19h
#     gpqa  avg@16 (198 items) : ~9h each reasoner                       ~= 20h
#   => avg@16 on everything is ~1.5-2 DAYS. So: math stays avg@16 (headline,
#   needs tight CI); GPQA drops to avg@4 (it's the control, expected weak).
#   Bump N_GPQA if you want, or move to the vLLM path for ~5-10x throughput.
set -euo pipefail
BASE="${BASE:-http://LLAMA_SWAP_HOST:8000}"
N_MATH=16
N_GPQA=4
MATH="aime,aime25,hmmt25"
COMMON="--base $BASE --concurrency 4 --max-tokens 40960"

case "${1:?usage: BASE=... ./run_all.sh vibe3b|base3b|qwen3think}" in
  vibe3b)
    python3 vibe_eval.py $COMMON --model vibethinker-3b --tag vibe3b      --n $N_MATH --temp 1.0 --cats $MATH
    python3 vibe_eval.py $COMMON --model vibethinker-3b --tag vibe3b_gpqa --n $N_GPQA --temp 1.0 --cats gpqa ;;
  base3b)
    python3 vibe_eval.py $COMMON --model base-3b --tag base3b      --n $N_MATH --temp 0.7 --cats $MATH
    python3 vibe_eval.py $COMMON --model base-3b --tag base3b_gpqa --n $N_GPQA --temp 0.7 --cats gpqa ;;
  qwen3think)
    python3 vibe_eval.py $COMMON --model qwen3-4b-think --tag qwen3think      --n $N_MATH --temp 0.6 --cats $MATH
    python3 vibe_eval.py $COMMON --model qwen3-4b-think --tag qwen3think_gpqa --n $N_GPQA --temp 0.6 --cats gpqa ;;
esac