diff --git a/11_start_server_122b.sh b/11_start_server_122b.sh index c922fc2..67b3b58 100755 --- a/11_start_server_122b.sh +++ b/11_start_server_122b.sh @@ -13,10 +13,13 @@ # Environment variables (override defaults): # MODEL_DIR - Path to model weights (default: ~/models/Qwen3.5-122B-A10B-FP8) # PORT - Server port (default: 7080) -# MAX_MODEL_LEN - Maximum context length (default: 32768) +# MAX_MODEL_LEN - Maximum context length (default: 131072 = 128k) # GPU_MEM_UTIL - GPU memory utilization fraction (default: 0.92) # API_KEY - API key for authentication (default: none) # TENSOR_PARALLEL - Number of GPUs for tensor parallelism (default: 4) +# MAX_NUM_SEQS - Max concurrent sequences (default: 4). Raise this +# when more users need it AND the KV cache fits; +# lower it if 128k context runs out of memory. # ------------------------------------------------------------------ set -euo pipefail @@ -25,10 +28,11 @@ SIF_FILE="${SCRIPT_DIR}/vllm_qwen.sif" MODEL_DIR="${MODEL_DIR:-$HOME/models/Qwen3.5-122B-A10B-FP8}" PORT="${PORT:-7080}" -MAX_MODEL_LEN="${MAX_MODEL_LEN:-32768}" +MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}" GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.92}" API_KEY="${API_KEY:-}" TENSOR_PARALLEL="${TENSOR_PARALLEL:-4}" +MAX_NUM_SEQS="${MAX_NUM_SEQS:-4}" if [ ! -f "$SIF_FILE" ]; then echo "ERROR: Container image not found at ${SIF_FILE}" @@ -56,7 +60,7 @@ VLLM_ARGS=( --trust-remote-code --reasoning-parser qwen3 --served-model-name "qwen3.5-122b-a10b-fp8" - --max-num-seqs 16 + --max-num-seqs "$MAX_NUM_SEQS" --enable-prefix-caching ) @@ -73,6 +77,7 @@ echo " Port: ${PORT}" echo " Context len: ${MAX_MODEL_LEN}" echo " GPU util: ${GPU_MEM_UTIL}" echo " TP size: ${TENSOR_PARALLEL}" +echo " Max seqs: ${MAX_NUM_SEQS}" echo " dtype: auto (FP8)" echo " API key: ${API_KEY:-}" echo "==============================================" diff --git a/13_check_server.sh b/13_check_server.sh new file mode 100755 index 0000000..d6f19fd --- /dev/null +++ b/13_check_server.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# ------------------------------------------------------------------ +# 13_check_server.sh +# Health-checks the vLLM server: polls /v1/models until the model is +# ready, then sends a properly-sized test prompt and prints the reply. +# +# Works whether you run it ON the server (localhost) or from another +# machine (use HOST=silicon.fhgr.ch). +# +# Usage: +# bash 13_check_server.sh +# HOST=silicon.fhgr.ch bash 13_check_server.sh +# +# Environment variables (override defaults): +# HOST - Server hostname (default: localhost) +# PORT - Server port (default: 7080) +# API_KEY - API key, if the server needs one (default: none) +# TIMEOUT - Seconds to wait for model (default: 600) +# MAX_TOKENS - Tokens for the test prompt (default: 512) +# +# NOTE: The model is a reasoning model — it spends tokens "thinking" +# before the final answer. MAX_TOKENS must be generous (>=256), +# or the reply ends up empty with finish_reason "length". +# ------------------------------------------------------------------ +set -euo pipefail + +HOST="${HOST:-localhost}" +PORT="${PORT:-7080}" +API_KEY="${API_KEY:-}" +TIMEOUT="${TIMEOUT:-600}" +MAX_TOKENS="${MAX_TOKENS:-512}" + +BASE_URL="http://${HOST}:${PORT}/v1" + +AUTH_ARGS=() +if [ -n "$API_KEY" ]; then + AUTH_ARGS=(-H "Authorization: Bearer ${API_KEY}") +fi + +echo "==============================================" +echo " vLLM Server Health Check" +echo "==============================================" +echo " Endpoint: ${BASE_URL}" +echo " Timeout: ${TIMEOUT}s" +echo " Max tokens: ${MAX_TOKENS}" +echo "==============================================" +echo "" + +# --- Step 1: poll /v1/models until ready ------------------------- +echo "Waiting for server to respond at ${BASE_URL}/models ..." +DEADLINE=$(( $(date +%s) + TIMEOUT )) +MODELS_JSON="" +while true; do + if MODELS_JSON=$(curl -s -m 5 ${AUTH_ARGS[@]+"${AUTH_ARGS[@]}"} "${BASE_URL}/models" 2>/dev/null) \ + && echo "$MODELS_JSON" | grep -q '"data"'; then + break + fi + if [ "$(date +%s)" -ge "$DEADLINE" ]; then + echo "ERROR: Server did not become ready within ${TIMEOUT}s." + echo " Check the log: tail -f logs/vllm_server_122b_*.log" + exit 1 + fi + printf '.' + sleep 5 +done +echo "" + +# Extract the served model id (first entry under data[]). +MODEL_ID=$(echo "$MODELS_JSON" | python3 -c \ + "import sys, json; print(json.load(sys.stdin)['data'][0]['id'])" 2>/dev/null || true) + +if [ -z "$MODEL_ID" ]; then + echo "ERROR: Could not parse model id from /v1/models response:" + echo "$MODELS_JSON" + exit 1 +fi + +echo "Server is up. Serving model: ${MODEL_ID}" +echo "" + +# --- Step 2: send a real generation request ---------------------- +echo "Sending test prompt..." +REQUEST=$(python3 -c " +import json, sys +print(json.dumps({ + 'model': sys.argv[1], + 'messages': [{'role': 'user', 'content': 'Reply with exactly one word: hello'}], + 'max_tokens': int(sys.argv[2]), +})) +" "$MODEL_ID" "$MAX_TOKENS") + +RESPONSE=$(curl -s -m 120 ${AUTH_ARGS[@]+"${AUTH_ARGS[@]}"} \ + -H "Content-Type: application/json" \ + -d "$REQUEST" \ + "${BASE_URL}/chat/completions") + +echo "" +echo "$RESPONSE" | python3 -c " +import sys, json +try: + d = json.load(sys.stdin) +except Exception: + print('ERROR: Non-JSON response from server:') + print(sys.stdin.read() if not sys.stdin.closed else '') + sys.exit(1) + +choice = d['choices'][0] +msg = choice.get('message', {}) +content = msg.get('content') +finish = choice.get('finish_reason') +usage = d.get('usage', {}) + +print('Finish reason: %s' % finish) +print('Usage: %s' % usage) +print('Content: %r' % content) +print() + +if content and content.strip(): + print('RESULT: OK — model generated a reply. Server is working.') +elif finish == 'length': + print('RESULT: Model hit the token limit while reasoning and never') + print(' produced an answer. Re-run with a larger budget, e.g.:') + print(' MAX_TOKENS=1024 bash 13_check_server.sh') + sys.exit(2) +else: + print('RESULT: Empty content with finish_reason=%r. Investigate.' % finish) + sys.exit(2) +"