ci/qwen-image
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

reorder

Browse Source
This commit is contained in:
mike
2026-06-27 00:39:32 +02:00
parent 61cd2e559b
commit 78ffb029b9
34 changed files with 22267 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
pose_llm/deploy_pose_llm.sh
View File

@@ -13,10 +13,14 @@ set -euo pipefail
REMOTE="tour@192.168.1.160"
REMOTE_DIR="/media/tour/NVME0/llm"
LLAMA_SERVER="$REMOTE_DIR/llama.cpp/build/bin/llama-server"
MODEL="$REMOTE_DIR/models/cognitivecomputations_Dolphin3.0-Mistral-24B-Q4_K_M.gguf"
# Q8_0: ~24GB — fits entirely on the MI50 32GB, better quality than Q4_K_M
# Override: MODEL_FILE=cognitivecomputations_Dolphin3.0-Mistral-24B-Q4_K_M.gguf ./deploy_pose_llm.sh deploy
MODEL_FILE="${MODEL_FILE:-cognitivecomputations_Dolphin3.0-Mistral-24B-Q8_0.gguf}"
MODEL_URL="https://huggingface.co/bartowski/cognitivecomputations_Dolphin3.0-Mistral-24B-GGUF/resolve/main/${MODEL_FILE}"
MODEL="$REMOTE_DIR/models/$MODEL_FILE"
PORT="${PORT:-8001}"
CTX="${CTX:-4096}"
NGL="${NGL:-99}" # GPU layers: 99 = all on GPU
CTX="${CTX:-16384}" # Q8 has headroom; bump context from 4096
NGL="${NGL:-99}" # all layers on GPU (24GB < 32GB VRAM)
ACTION="${1:-deploy}"
@@ -42,8 +46,23 @@ stop_one() {
"
}
download_model() {
print_header "Downloading $MODEL_FILE to $REMOTE"
ssh "$REMOTE" "
set -euo pipefail
if [ -f '$MODEL' ]; then
echo '==> Already exists:'; ls -lh '$MODEL'; exit 0
fi
echo '==> Downloading ~24GB — this will take a while...'
mkdir -p '$REMOTE_DIR/models'
wget -c --show-progress -O '${MODEL}.tmp' '$MODEL_URL'
mv '${MODEL}.tmp' '$MODEL'
echo '==> Done:'; ls -lh '$MODEL'
"
}
deploy_one() {
print_header "Deploying llama-server on $REMOTE (model: Q4_K_M, ngl=$NGL)"
print_header "Deploying llama-server on $REMOTE (model: Q8_0, ngl=$NGL)"
echo "==> Writing start script..."
ssh "$REMOTE" "cat > '$REMOTE_DIR/start_pose_llm.sh'" << EOF
@@ -96,9 +115,10 @@ status_one() {
}
case "$ACTION" in
deploy) deploy_one ;;
stop) stop_one ;;
restart) stop_one; deploy_one ;;
status) status_one ;;
*) echo "Usage: $0 [deploy|stop|restart|status]"; exit 1 ;;
deploy) download_model; deploy_one ;;
stop) stop_one ;;
restart) stop_one; deploy_one ;;
status) status_one ;;
download) download_model ;;
*) echo "Usage: $0 [deploy|stop|restart|status|download]"; exit 1 ;;
esac