125 lines
4.5 KiB
Bash
Executable File
125 lines
4.5 KiB
Bash
Executable File
#!/bin/bash
|
|
# Deploy/stop/restart/status the uncensored chat LLM API on tour.
|
|
# Serves Dolphin-24B GGUF via llama-server (full GPU, ~10-20 tok/s).
|
|
#
|
|
# Usage:
|
|
# ./deploy_pose_llm.sh deploy # (re)start, wait for health
|
|
# ./deploy_pose_llm.sh stop
|
|
# ./deploy_pose_llm.sh restart
|
|
# ./deploy_pose_llm.sh status
|
|
|
|
set -euo pipefail
|
|
|
|
REMOTE="tour@192.168.1.160"
|
|
REMOTE_DIR="/media/tour/NVME0/llm"
|
|
LLAMA_SERVER="$REMOTE_DIR/llama.cpp/build/bin/llama-server"
|
|
# Q8_0: ~24GB — fits entirely on the MI50 32GB, better quality than Q4_K_M
|
|
# Override: MODEL_FILE=cognitivecomputations_Dolphin3.0-Mistral-24B-Q4_K_M.gguf ./deploy_pose_llm.sh deploy
|
|
MODEL_FILE="${MODEL_FILE:-cognitivecomputations_Dolphin3.0-Mistral-24B-Q8_0.gguf}"
|
|
MODEL_URL="https://huggingface.co/bartowski/cognitivecomputations_Dolphin3.0-Mistral-24B-GGUF/resolve/main/${MODEL_FILE}"
|
|
MODEL="$REMOTE_DIR/models/$MODEL_FILE"
|
|
PORT="${PORT:-8001}"
|
|
CTX="${CTX:-16384}" # Q8 has headroom; bump context from 4096
|
|
NGL="${NGL:-99}" # all layers on GPU (24GB < 32GB VRAM)
|
|
|
|
ACTION="${1:-deploy}"
|
|
|
|
print_header() { echo; echo "============================================================"; echo "$1"; echo "============================================================"; }
|
|
|
|
stop_one() {
|
|
print_header "Stopping pose-LLM on $REMOTE"
|
|
ssh "$REMOTE" "
|
|
cd '$REMOTE_DIR'
|
|
if [ -f api.pid ]; then
|
|
PID=\$(cat api.pid)
|
|
if kill -0 \"\$PID\" 2>/dev/null; then
|
|
echo '==> Stopping PID' \"\$PID\"; kill \"\$PID\"
|
|
for i in \$(seq 1 30); do kill -0 \"\$PID\" 2>/dev/null || { echo '==> stopped'; rm -f api.pid; exit 0; }; sleep 1; done
|
|
echo '==> hard kill'; kill -9 \"\$PID\" 2>/dev/null || true
|
|
fi
|
|
rm -f api.pid
|
|
else echo '==> no api.pid'; fi
|
|
PORT_PID=\$(lsof -ti:$PORT 2>/dev/null || true)
|
|
[ -n \"\$PORT_PID\" ] && { echo '==> killing port $PORT:' \"\$PORT_PID\"; kill \$PORT_PID 2>/dev/null || true; }
|
|
echo '==> waiting 8s for VRAM drain...'; sleep 8
|
|
true
|
|
"
|
|
}
|
|
|
|
download_model() {
|
|
print_header "Downloading $MODEL_FILE to $REMOTE"
|
|
ssh "$REMOTE" "
|
|
set -euo pipefail
|
|
if [ -f '$MODEL' ]; then
|
|
echo '==> Already exists:'; ls -lh '$MODEL'; exit 0
|
|
fi
|
|
echo '==> Downloading ~24GB — this will take a while...'
|
|
mkdir -p '$REMOTE_DIR/models'
|
|
wget -c --show-progress -O '${MODEL}.tmp' '$MODEL_URL'
|
|
mv '${MODEL}.tmp' '$MODEL'
|
|
echo '==> Done:'; ls -lh '$MODEL'
|
|
"
|
|
}
|
|
|
|
deploy_one() {
|
|
print_header "Deploying llama-server on $REMOTE (model: Q8_0, ngl=$NGL)"
|
|
|
|
echo "==> Writing start script..."
|
|
ssh "$REMOTE" "cat > '$REMOTE_DIR/start_pose_llm.sh'" << EOF
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
cd "$REMOTE_DIR"
|
|
if [ -f api.pid ]; then
|
|
kill \$(cat api.pid) 2>/dev/null || true
|
|
rm -f api.pid
|
|
echo "waiting for VRAM to drain..."
|
|
sleep 10
|
|
fi
|
|
export HSA_OVERRIDE_GFX_VERSION=9.0.6
|
|
export ROCR_VISIBLE_DEVICES=0
|
|
nohup $LLAMA_SERVER \\
|
|
-m $MODEL \\
|
|
--host 0.0.0.0 \\
|
|
--port $PORT \\
|
|
-ngl $NGL \\
|
|
-c $CTX \\
|
|
--chat-template chatml \\
|
|
-np 1 \\
|
|
> api.log 2>&1 &
|
|
echo \$! > api.pid
|
|
echo "started PID \$(cat api.pid)"
|
|
EOF
|
|
ssh "$REMOTE" "chmod +x '$REMOTE_DIR/start_pose_llm.sh' && '$REMOTE_DIR/start_pose_llm.sh'"
|
|
|
|
echo "==> Waiting for model load + health (can take ~30s)..."
|
|
for i in $(seq 1 90); do
|
|
if ssh "$REMOTE" "curl -fsS http://localhost:$PORT/health >/dev/null 2>&1"; then echo "==> API ready"; break; fi
|
|
if [ "$i" -eq 90 ]; then echo "ERROR: not healthy after 180s"; ssh "$REMOTE" "tail -n 60 $REMOTE_DIR/api.log"; exit 1; fi
|
|
sleep 2
|
|
done
|
|
echo "==> Health:"; ssh "$REMOTE" "curl -s http://localhost:$PORT/health | python3 -m json.tool"
|
|
}
|
|
|
|
status_one() {
|
|
print_header "Status on $REMOTE"
|
|
ssh "$REMOTE" "
|
|
cd '$REMOTE_DIR'
|
|
echo '==> Binary:'; ls -lh $LLAMA_SERVER 2>/dev/null || echo 'NOT BUILT'
|
|
echo '==> Model:'; ls -lh $MODEL 2>/dev/null || echo 'NOT DOWNLOADED'
|
|
echo '==> PID:'; cat api.pid 2>/dev/null || echo none
|
|
echo '==> Port $PORT:'; lsof -i:$PORT 2>/dev/null || echo 'nothing listening'
|
|
echo '==> Health:'; curl -fsS http://localhost:$PORT/health 2>/dev/null | python3 -m json.tool || echo 'not healthy'
|
|
echo '==> Last 20 log lines:'; tail -n 20 api.log 2>/dev/null || true
|
|
echo '==> VRAM:'; /opt/rocm/bin/rocm-smi --showmeminfo vram 2>/dev/null | grep -i 'used\|total' || true
|
|
"
|
|
}
|
|
|
|
case "$ACTION" in
|
|
deploy) download_model; deploy_one ;;
|
|
stop) stop_one ;;
|
|
restart) stop_one; deploy_one ;;
|
|
status) status_one ;;
|
|
download) download_model ;;
|
|
*) echo "Usage: $0 [deploy|stop|restart|status|download]"; exit 1 ;;
|
|
esac
|