#!/bin/bash
# Deploy/stop/restart/status the uncensored chat LLM API on tour.
# Serves Dolphin-24B GGUF via llama-server (full GPU, ~10-20 tok/s).
#
# Usage:
#   ./deploy_pose_llm.sh deploy     # (re)start, wait for health
#   ./deploy_pose_llm.sh stop
#   ./deploy_pose_llm.sh restart
#   ./deploy_pose_llm.sh status

set -euo pipefail

REMOTE="tour@192.168.1.160"
REMOTE_DIR="/media/tour/NVME0/llm"
LLAMA_SERVER="$REMOTE_DIR/llama.cpp/build/bin/llama-server"
MODEL="$REMOTE_DIR/models/cognitivecomputations_Dolphin3.0-Mistral-24B-Q4_K_M.gguf"
PORT="${PORT:-8001}"
CTX="${CTX:-4096}"
NGL="${NGL:-99}"  # GPU layers: 99 = all on GPU

ACTION="${1:-deploy}"

print_header() { echo; echo "============================================================"; echo "$1"; echo "============================================================"; }

stop_one() {
  print_header "Stopping pose-LLM on $REMOTE"
  ssh "$REMOTE" "
    cd '$REMOTE_DIR'
    if [ -f api.pid ]; then
      PID=\$(cat api.pid)
      if kill -0 \"\$PID\" 2>/dev/null; then
        echo '==> Stopping PID' \"\$PID\"; kill \"\$PID\"
        for i in \$(seq 1 30); do kill -0 \"\$PID\" 2>/dev/null || { echo '==> stopped'; rm -f api.pid; exit 0; }; sleep 1; done
        echo '==> hard kill'; kill -9 \"\$PID\" 2>/dev/null || true
      fi
      rm -f api.pid
    else echo '==> no api.pid'; fi
    PORT_PID=\$(lsof -ti:$PORT 2>/dev/null || true)
    [ -n \"\$PORT_PID\" ] && { echo '==> killing port $PORT:' \"\$PORT_PID\"; kill \$PORT_PID 2>/dev/null || true; }
    echo '==> waiting 8s for VRAM drain...'; sleep 8
    true
  "
}

deploy_one() {
  print_header "Deploying llama-server on $REMOTE (model: Q4_K_M, ngl=$NGL)"

  echo "==> Writing start script..."
  ssh "$REMOTE" "cat > '$REMOTE_DIR/start_pose_llm.sh'" << EOF
#!/usr/bin/env bash
set -euo pipefail
cd "$REMOTE_DIR"
if [ -f api.pid ]; then
  kill \$(cat api.pid) 2>/dev/null || true
  rm -f api.pid
  echo "waiting for VRAM to drain..."
  sleep 10
fi
export HSA_OVERRIDE_GFX_VERSION=9.0.6
export ROCR_VISIBLE_DEVICES=0
nohup $LLAMA_SERVER \\
  -m $MODEL \\
  --host 0.0.0.0 \\
  --port $PORT \\
  -ngl $NGL \\
  -c $CTX \\
  --chat-template chatml \\
  -np 1 \\
  > api.log 2>&1 &
echo \$! > api.pid
echo "started PID \$(cat api.pid)"
EOF
  ssh "$REMOTE" "chmod +x '$REMOTE_DIR/start_pose_llm.sh' && '$REMOTE_DIR/start_pose_llm.sh'"

  echo "==> Waiting for model load + health (can take ~30s)..."
  for i in $(seq 1 90); do
    if ssh "$REMOTE" "curl -fsS http://localhost:$PORT/health >/dev/null 2>&1"; then echo "==> API ready"; break; fi
    if [ "$i" -eq 90 ]; then echo "ERROR: not healthy after 180s"; ssh "$REMOTE" "tail -n 60 $REMOTE_DIR/api.log"; exit 1; fi
    sleep 2
  done
  echo "==> Health:"; ssh "$REMOTE" "curl -s http://localhost:$PORT/health | python3 -m json.tool"
}

status_one() {
  print_header "Status on $REMOTE"
  ssh "$REMOTE" "
    cd '$REMOTE_DIR'
    echo '==> Binary:'; ls -lh $LLAMA_SERVER 2>/dev/null || echo 'NOT BUILT'
    echo '==> Model:'; ls -lh $MODEL 2>/dev/null || echo 'NOT DOWNLOADED'
    echo '==> PID:'; cat api.pid 2>/dev/null || echo none
    echo '==> Port $PORT:'; lsof -i:$PORT 2>/dev/null || echo 'nothing listening'
    echo '==> Health:'; curl -fsS http://localhost:$PORT/health 2>/dev/null | python3 -m json.tool || echo 'not healthy'
    echo '==> Last 20 log lines:'; tail -n 20 api.log 2>/dev/null || true
    echo '==> VRAM:'; /opt/rocm/bin/rocm-smi --showmeminfo vram 2>/dev/null | grep -i 'used\|total' || true
  "
}

case "$ACTION" in
  deploy)  deploy_one ;;
  stop)    stop_one ;;
  restart) stop_one; deploy_one ;;
  status)  status_one ;;
  *) echo "Usage: $0 [deploy|stop|restart|status]"; exit 1 ;;
esac